test_check.py 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596
  1. import pytest
  2. from flexmock import flexmock
  3. from borgmatic.actions import check as module
  4. from borgmatic.borg.pattern import Pattern
  5. def test_parse_checks_returns_them_as_tuple():
  6. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'bar'}]})
  7. assert checks == ('foo', 'bar')
  8. def test_parse_checks_with_missing_value_returns_defaults():
  9. checks = module.parse_checks({})
  10. assert checks == ('repository', 'archives')
  11. def test_parse_checks_with_empty_list_returns_defaults():
  12. checks = module.parse_checks({'checks': []})
  13. assert checks == ('repository', 'archives')
  14. def test_parse_checks_with_none_value_returns_defaults():
  15. checks = module.parse_checks({'checks': None})
  16. assert checks == ('repository', 'archives')
  17. def test_parse_checks_with_disabled_returns_no_checks():
  18. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'disabled'}]})
  19. assert checks == ()
  20. def test_parse_checks_prefers_override_checks_to_configured_checks():
  21. checks = module.parse_checks(
  22. {'checks': [{'name': 'archives'}]}, only_checks=['repository', 'extract']
  23. )
  24. assert checks == ('repository', 'extract')
  25. @pytest.mark.parametrize(
  26. 'frequency,expected_result',
  27. (
  28. (None, None),
  29. ('always', None),
  30. ('1 hour', module.datetime.timedelta(hours=1)),
  31. ('2 hours', module.datetime.timedelta(hours=2)),
  32. ('1 day', module.datetime.timedelta(days=1)),
  33. ('2 days', module.datetime.timedelta(days=2)),
  34. ('1 week', module.datetime.timedelta(weeks=1)),
  35. ('2 weeks', module.datetime.timedelta(weeks=2)),
  36. ('1 month', module.datetime.timedelta(days=30)),
  37. ('2 months', module.datetime.timedelta(days=60)),
  38. ('1 year', module.datetime.timedelta(days=365)),
  39. ('2 years', module.datetime.timedelta(days=365 * 2)),
  40. ),
  41. )
  42. def test_parse_frequency_parses_into_timedeltas(frequency, expected_result):
  43. assert module.parse_frequency(frequency) == expected_result
  44. @pytest.mark.parametrize(
  45. 'frequency',
  46. (
  47. 'sometime',
  48. 'x days',
  49. '3 decades',
  50. ),
  51. )
  52. def test_parse_frequency_raises_on_parse_error(frequency):
  53. with pytest.raises(ValueError):
  54. module.parse_frequency(frequency)
  55. def test_filter_checks_on_frequency_without_config_uses_default_checks():
  56. flexmock(module).should_receive('parse_frequency').and_return(
  57. module.datetime.timedelta(weeks=4)
  58. )
  59. flexmock(module).should_receive('make_check_time_path')
  60. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  61. assert module.filter_checks_on_frequency(
  62. config={},
  63. borg_repository_id='repo',
  64. checks=('repository', 'archives'),
  65. force=False,
  66. archives_check_id='1234',
  67. ) == ('repository', 'archives')
  68. def test_filter_checks_on_frequency_retains_unconfigured_check():
  69. assert module.filter_checks_on_frequency(
  70. config={},
  71. borg_repository_id='repo',
  72. checks=('data',),
  73. force=False,
  74. ) == ('data',)
  75. def test_filter_checks_on_frequency_retains_check_without_frequency():
  76. flexmock(module).should_receive('parse_frequency').and_return(None)
  77. assert module.filter_checks_on_frequency(
  78. config={'checks': [{'name': 'archives'}]},
  79. borg_repository_id='repo',
  80. checks=('archives',),
  81. force=False,
  82. archives_check_id='1234',
  83. ) == ('archives',)
  84. def test_filter_checks_on_frequency_retains_check_with_empty_only_run_on():
  85. flexmock(module).should_receive('parse_frequency').and_return(None)
  86. assert module.filter_checks_on_frequency(
  87. config={'checks': [{'name': 'archives', 'only_run_on': []}]},
  88. borg_repository_id='repo',
  89. checks=('archives',),
  90. force=False,
  91. archives_check_id='1234',
  92. datetime_now=flexmock(weekday=lambda: 0),
  93. ) == ('archives',)
  94. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today():
  95. flexmock(module).should_receive('parse_frequency').and_return(None)
  96. assert module.filter_checks_on_frequency(
  97. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[0]]}]},
  98. borg_repository_id='repo',
  99. checks=('archives',),
  100. force=False,
  101. archives_check_id='1234',
  102. datetime_now=flexmock(weekday=lambda: 0),
  103. ) == ('archives',)
  104. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekday_value():
  105. flexmock(module).should_receive('parse_frequency').and_return(None)
  106. assert module.filter_checks_on_frequency(
  107. config={'checks': [{'name': 'archives', 'only_run_on': ['weekday']}]},
  108. borg_repository_id='repo',
  109. checks=('archives',),
  110. force=False,
  111. archives_check_id='1234',
  112. datetime_now=flexmock(weekday=lambda: 0),
  113. ) == ('archives',)
  114. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekend_value():
  115. flexmock(module).should_receive('parse_frequency').and_return(None)
  116. assert module.filter_checks_on_frequency(
  117. config={'checks': [{'name': 'archives', 'only_run_on': ['weekend']}]},
  118. borg_repository_id='repo',
  119. checks=('archives',),
  120. force=False,
  121. archives_check_id='1234',
  122. datetime_now=flexmock(weekday=lambda: 6),
  123. ) == ('archives',)
  124. def test_filter_checks_on_frequency_skips_check_with_only_run_on_not_matching_today():
  125. flexmock(module).should_receive('parse_frequency').and_return(None)
  126. assert (
  127. module.filter_checks_on_frequency(
  128. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[5]]}]},
  129. borg_repository_id='repo',
  130. checks=('archives',),
  131. force=False,
  132. archives_check_id='1234',
  133. datetime_now=flexmock(weekday=lambda: 0),
  134. )
  135. == ()
  136. )
  137. def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
  138. flexmock(module).should_receive('parse_frequency').and_return(
  139. module.datetime.timedelta(hours=1)
  140. )
  141. flexmock(module).should_receive('make_check_time_path')
  142. flexmock(module).should_receive('probe_for_check_time').and_return(
  143. module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1)
  144. )
  145. assert module.filter_checks_on_frequency(
  146. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  147. borg_repository_id='repo',
  148. checks=('archives',),
  149. force=False,
  150. archives_check_id='1234',
  151. ) == ('archives',)
  152. def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file():
  153. flexmock(module).should_receive('parse_frequency').and_return(
  154. module.datetime.timedelta(hours=1)
  155. )
  156. flexmock(module).should_receive('make_check_time_path')
  157. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  158. assert module.filter_checks_on_frequency(
  159. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  160. borg_repository_id='repo',
  161. checks=('archives',),
  162. force=False,
  163. archives_check_id='1234',
  164. ) == ('archives',)
  165. def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
  166. flexmock(module).should_receive('parse_frequency').and_return(
  167. module.datetime.timedelta(hours=1)
  168. )
  169. flexmock(module).should_receive('make_check_time_path')
  170. flexmock(module).should_receive('probe_for_check_time').and_return(
  171. module.datetime.datetime.now()
  172. )
  173. assert (
  174. module.filter_checks_on_frequency(
  175. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  176. borg_repository_id='repo',
  177. checks=('archives',),
  178. force=False,
  179. archives_check_id='1234',
  180. )
  181. == ()
  182. )
  183. def test_filter_checks_on_frequency_retains_check_with_unelapsed_frequency_and_force():
  184. assert module.filter_checks_on_frequency(
  185. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  186. borg_repository_id='repo',
  187. checks=('archives',),
  188. force=True,
  189. archives_check_id='1234',
  190. ) == ('archives',)
  191. def test_filter_checks_on_frequency_passes_through_empty_checks():
  192. assert (
  193. module.filter_checks_on_frequency(
  194. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  195. borg_repository_id='repo',
  196. checks=(),
  197. force=False,
  198. archives_check_id='1234',
  199. )
  200. == ()
  201. )
  202. def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise():
  203. assert module.make_archives_check_id(('--match-archives', 'sh:foo-*'))
  204. def test_make_archives_check_id_with_empty_flags_returns_none():
  205. assert module.make_archives_check_id(()) is None
  206. def test_make_check_time_path_with_borgmatic_source_directory_includes_it():
  207. flexmock(module.borgmatic.config.paths).should_receive(
  208. 'get_borgmatic_state_directory'
  209. ).and_return(
  210. '/home/user/.local/state/borgmatic',
  211. )
  212. assert (
  213. module.make_check_time_path({}, '1234', 'archives', '5678')
  214. == '/home/user/.local/state/borgmatic/checks/1234/archives/5678'
  215. )
  216. def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all():
  217. flexmock(module.borgmatic.config.paths).should_receive(
  218. 'get_borgmatic_state_directory'
  219. ).and_return(
  220. '/home/user/.local/state/borgmatic',
  221. )
  222. assert (
  223. module.make_check_time_path(
  224. {},
  225. '1234',
  226. 'archives',
  227. )
  228. == '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  229. )
  230. def test_make_check_time_path_with_repositories_check_ignores_archives_check_id():
  231. flexmock(module.borgmatic.config.paths).should_receive(
  232. 'get_borgmatic_state_directory'
  233. ).and_return(
  234. '/home/user/.local/state/borgmatic',
  235. )
  236. assert (
  237. module.make_check_time_path({}, '1234', 'repository', '5678')
  238. == '/home/user/.local/state/borgmatic/checks/1234/repository'
  239. )
  240. def test_read_check_time_does_not_raise():
  241. flexmock(module.os).should_receive('stat').and_return(flexmock(st_mtime=123))
  242. assert module.read_check_time('/path')
  243. def test_read_check_time_on_missing_file_does_not_raise():
  244. flexmock(module.os).should_receive('stat').and_raise(FileNotFoundError)
  245. assert module.read_check_time('/path') is None
  246. def test_probe_for_check_time_uses_maximum_of_multiple_check_times():
  247. flexmock(module).should_receive('make_check_time_path').and_return(
  248. '~/.borgmatic/checks/1234/archives/5678'
  249. ).and_return('~/.borgmatic/checks/1234/archives/all')
  250. flexmock(module).should_receive('read_check_time').and_return(1).and_return(2)
  251. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  252. def test_probe_for_check_time_deduplicates_identical_check_time_paths():
  253. flexmock(module).should_receive('make_check_time_path').and_return(
  254. '~/.borgmatic/checks/1234/archives/5678'
  255. ).and_return('~/.borgmatic/checks/1234/archives/5678')
  256. flexmock(module).should_receive('read_check_time').and_return(1).once()
  257. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  258. def test_probe_for_check_time_skips_none_check_time():
  259. flexmock(module).should_receive('make_check_time_path').and_return(
  260. '~/.borgmatic/checks/1234/archives/5678'
  261. ).and_return('~/.borgmatic/checks/1234/archives/all')
  262. flexmock(module).should_receive('read_check_time').and_return(None).and_return(2)
  263. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  264. def test_probe_for_check_time_uses_single_check_time():
  265. flexmock(module).should_receive('make_check_time_path').and_return(
  266. '~/.borgmatic/checks/1234/archives/5678'
  267. ).and_return('~/.borgmatic/checks/1234/archives/all')
  268. flexmock(module).should_receive('read_check_time').and_return(1).and_return(None)
  269. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  270. def test_probe_for_check_time_returns_none_when_no_check_time_found():
  271. flexmock(module).should_receive('make_check_time_path').and_return(
  272. '~/.borgmatic/checks/1234/archives/5678'
  273. ).and_return('~/.borgmatic/checks/1234/archives/all')
  274. flexmock(module).should_receive('read_check_time').and_return(None).and_return(None)
  275. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None
  276. def test_upgrade_check_times_moves_checks_from_borgmatic_source_directory_to_state_directory():
  277. flexmock(module.borgmatic.config.paths).should_receive(
  278. 'get_borgmatic_source_directory'
  279. ).and_return('/home/user/.borgmatic')
  280. flexmock(module.borgmatic.config.paths).should_receive(
  281. 'get_borgmatic_state_directory'
  282. ).and_return('/home/user/.local/state/borgmatic')
  283. flexmock(module.os.path).should_receive('exists').with_args(
  284. '/home/user/.borgmatic/checks'
  285. ).and_return(True)
  286. flexmock(module.os.path).should_receive('exists').with_args(
  287. '/home/user/.local/state/borgmatic/checks'
  288. ).and_return(False)
  289. flexmock(module.os).should_receive('makedirs')
  290. flexmock(module.shutil).should_receive('move').with_args(
  291. '/home/user/.borgmatic/checks', '/home/user/.local/state/borgmatic/checks'
  292. ).once()
  293. flexmock(module).should_receive('make_check_time_path').and_return(
  294. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  295. )
  296. flexmock(module.os.path).should_receive('isfile').and_return(False)
  297. flexmock(module.os).should_receive('mkdir').never()
  298. module.upgrade_check_times(flexmock(), flexmock())
  299. def test_upgrade_check_times_with_checks_already_in_borgmatic_state_directory_does_not_move_anything():
  300. flexmock(module.borgmatic.config.paths).should_receive(
  301. 'get_borgmatic_source_directory'
  302. ).and_return('/home/user/.borgmatic')
  303. flexmock(module.borgmatic.config.paths).should_receive(
  304. 'get_borgmatic_state_directory'
  305. ).and_return('/home/user/.local/state/borgmatic')
  306. flexmock(module.os.path).should_receive('exists').with_args(
  307. '/home/user/.borgmatic/checks'
  308. ).and_return(True)
  309. flexmock(module.os.path).should_receive('exists').with_args(
  310. '/home/user/.local/state/borgmatic/checks'
  311. ).and_return(True)
  312. flexmock(module.os).should_receive('makedirs').never()
  313. flexmock(module.shutil).should_receive('move').never()
  314. flexmock(module).should_receive('make_check_time_path').and_return(
  315. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  316. )
  317. flexmock(module.os.path).should_receive('isfile').and_return(False)
  318. flexmock(module.shutil).should_receive('move').never()
  319. flexmock(module.os).should_receive('mkdir').never()
  320. module.upgrade_check_times(flexmock(), flexmock())
  321. def test_upgrade_check_times_renames_old_check_paths_to_all():
  322. flexmock(module.borgmatic.config.paths).should_receive(
  323. 'get_borgmatic_source_directory'
  324. ).and_return('/home/user/.borgmatic')
  325. flexmock(module.borgmatic.config.paths).should_receive(
  326. 'get_borgmatic_state_directory'
  327. ).and_return('/home/user/.local/state/borgmatic')
  328. flexmock(module.os.path).should_receive('exists').and_return(False)
  329. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  330. flexmock(module).should_receive('make_check_time_path').with_args(
  331. object, object, 'archives', 'all'
  332. ).and_return(f'{base_path}/archives/all')
  333. flexmock(module).should_receive('make_check_time_path').with_args(
  334. object, object, 'data', 'all'
  335. ).and_return(f'{base_path}/data/all')
  336. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  337. True
  338. )
  339. flexmock(module.os.path).should_receive('isfile').with_args(
  340. f'{base_path}/archives.temp'
  341. ).and_return(False)
  342. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  343. False
  344. )
  345. flexmock(module.os.path).should_receive('isfile').with_args(
  346. f'{base_path}/data.temp'
  347. ).and_return(False)
  348. flexmock(module.shutil).should_receive('move').with_args(
  349. f'{base_path}/archives', f'{base_path}/archives.temp'
  350. ).once()
  351. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  352. flexmock(module.shutil).should_receive('move').with_args(
  353. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  354. ).once()
  355. module.upgrade_check_times(flexmock(), flexmock())
  356. def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded():
  357. flexmock(module.borgmatic.config.paths).should_receive(
  358. 'get_borgmatic_source_directory'
  359. ).and_return('/home/user/.borgmatic')
  360. flexmock(module.borgmatic.config.paths).should_receive(
  361. 'get_borgmatic_state_directory'
  362. ).and_return('/home/user/.local/state/borgmatic')
  363. flexmock(module.os.path).should_receive('exists').and_return(False)
  364. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  365. flexmock(module).should_receive('make_check_time_path').with_args(
  366. object, object, 'archives', 'all'
  367. ).and_return(f'{base_path}/archives/all')
  368. flexmock(module).should_receive('make_check_time_path').with_args(
  369. object, object, 'data', 'all'
  370. ).and_return(f'{base_path}/data/all')
  371. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  372. False
  373. )
  374. flexmock(module.os.path).should_receive('isfile').with_args(
  375. f'{base_path}/archives.temp'
  376. ).and_return(False)
  377. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  378. True
  379. )
  380. flexmock(module.shutil).should_receive('move').with_args(
  381. f'{base_path}/data', f'{base_path}/data.temp'
  382. ).once()
  383. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/data').once()
  384. flexmock(module.shutil).should_receive('move').with_args(
  385. f'{base_path}/data.temp', f'{base_path}/data/all'
  386. ).once()
  387. module.upgrade_check_times(flexmock(), flexmock())
  388. def test_upgrade_check_times_skips_already_upgraded_check_paths():
  389. flexmock(module.borgmatic.config.paths).should_receive(
  390. 'get_borgmatic_source_directory'
  391. ).and_return('/home/user/.borgmatic')
  392. flexmock(module.borgmatic.config.paths).should_receive(
  393. 'get_borgmatic_state_directory'
  394. ).and_return('/home/user/.local/state/borgmatic')
  395. flexmock(module.os.path).should_receive('exists').and_return(False)
  396. flexmock(module).should_receive('make_check_time_path').and_return(
  397. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  398. )
  399. flexmock(module.os.path).should_receive('isfile').and_return(False)
  400. flexmock(module.shutil).should_receive('move').never()
  401. flexmock(module.os).should_receive('mkdir').never()
  402. module.upgrade_check_times(flexmock(), flexmock())
  403. def test_upgrade_check_times_renames_stale_temporary_check_path():
  404. flexmock(module.borgmatic.config.paths).should_receive(
  405. 'get_borgmatic_source_directory'
  406. ).and_return('/home/user/.borgmatic')
  407. flexmock(module.borgmatic.config.paths).should_receive(
  408. 'get_borgmatic_state_directory'
  409. ).and_return('/home/user/.local/state/borgmatic')
  410. flexmock(module.os.path).should_receive('exists').and_return(False)
  411. base_path = '/home/borgmatic/.local/state/checks/1234'
  412. flexmock(module).should_receive('make_check_time_path').with_args(
  413. object, object, 'archives', 'all'
  414. ).and_return(f'{base_path}/archives/all')
  415. flexmock(module).should_receive('make_check_time_path').with_args(
  416. object, object, 'data', 'all'
  417. ).and_return(f'{base_path}/data/all')
  418. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  419. False
  420. )
  421. flexmock(module.os.path).should_receive('isfile').with_args(
  422. f'{base_path}/archives.temp'
  423. ).and_return(True)
  424. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  425. False
  426. )
  427. flexmock(module.os.path).should_receive('isfile').with_args(
  428. f'{base_path}/data.temp'
  429. ).and_return(False)
  430. flexmock(module.shutil).should_receive('move').with_args(
  431. f'{base_path}/archives', f'{base_path}/archives.temp'
  432. ).and_raise(FileNotFoundError)
  433. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  434. flexmock(module.shutil).should_receive('move').with_args(
  435. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  436. ).once()
  437. module.upgrade_check_times(flexmock(), flexmock())
  438. def test_collect_spot_check_source_paths_parses_borg_output():
  439. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  440. {'hook1': False, 'hook2': True}
  441. )
  442. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  443. flexmock()
  444. )
  445. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  446. flexmock()
  447. )
  448. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  449. [Pattern('foo'), Pattern('bar')]
  450. )
  451. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  452. dry_run=True,
  453. repository_path='repo',
  454. config=object,
  455. patterns=[Pattern('foo'), Pattern('bar')],
  456. local_borg_version=object,
  457. global_arguments=object,
  458. borgmatic_runtime_directory='/run/borgmatic',
  459. local_path=object,
  460. remote_path=object,
  461. list_files=True,
  462. stream_processes=True,
  463. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  464. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  465. flexmock()
  466. )
  467. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  468. flexmock(module.borgmatic.execute).should_receive(
  469. 'execute_command_and_capture_output'
  470. ).and_return(
  471. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  472. )
  473. flexmock(module.os.path).should_receive('isfile').and_return(True)
  474. assert module.collect_spot_check_source_paths(
  475. repository={'path': 'repo'},
  476. config={'working_directory': '/'},
  477. local_borg_version=flexmock(),
  478. global_arguments=flexmock(),
  479. local_path=flexmock(),
  480. remote_path=flexmock(),
  481. borgmatic_runtime_directory='/run/borgmatic',
  482. ) == ('/etc/path', '/etc/other')
  483. def test_collect_spot_check_source_paths_passes_through_stream_processes_false():
  484. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  485. {'hook1': False, 'hook2': False}
  486. )
  487. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  488. flexmock()
  489. )
  490. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  491. flexmock()
  492. )
  493. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  494. [Pattern('foo'), Pattern('bar')]
  495. )
  496. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  497. dry_run=True,
  498. repository_path='repo',
  499. config=object,
  500. patterns=[Pattern('foo'), Pattern('bar')],
  501. local_borg_version=object,
  502. global_arguments=object,
  503. borgmatic_runtime_directory='/run/borgmatic',
  504. local_path=object,
  505. remote_path=object,
  506. list_files=True,
  507. stream_processes=False,
  508. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  509. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  510. flexmock()
  511. )
  512. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  513. flexmock(module.borgmatic.execute).should_receive(
  514. 'execute_command_and_capture_output'
  515. ).and_return(
  516. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  517. )
  518. flexmock(module.os.path).should_receive('isfile').and_return(True)
  519. assert module.collect_spot_check_source_paths(
  520. repository={'path': 'repo'},
  521. config={'working_directory': '/'},
  522. local_borg_version=flexmock(),
  523. global_arguments=flexmock(),
  524. local_path=flexmock(),
  525. remote_path=flexmock(),
  526. borgmatic_runtime_directory='/run/borgmatic',
  527. ) == ('/etc/path', '/etc/other')
  528. def test_collect_spot_check_source_paths_without_working_directory_parses_borg_output():
  529. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  530. {'hook1': False, 'hook2': True}
  531. )
  532. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  533. flexmock()
  534. )
  535. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  536. flexmock()
  537. )
  538. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  539. [Pattern('foo'), Pattern('bar')]
  540. )
  541. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  542. dry_run=True,
  543. repository_path='repo',
  544. config=object,
  545. patterns=[Pattern('foo'), Pattern('bar')],
  546. local_borg_version=object,
  547. global_arguments=object,
  548. borgmatic_runtime_directory='/run/borgmatic',
  549. local_path=object,
  550. remote_path=object,
  551. list_files=True,
  552. stream_processes=True,
  553. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  554. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  555. flexmock()
  556. )
  557. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  558. flexmock(module.borgmatic.execute).should_receive(
  559. 'execute_command_and_capture_output'
  560. ).and_return(
  561. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  562. )
  563. flexmock(module.os.path).should_receive('isfile').and_return(True)
  564. assert module.collect_spot_check_source_paths(
  565. repository={'path': 'repo'},
  566. config={},
  567. local_borg_version=flexmock(),
  568. global_arguments=flexmock(),
  569. local_path=flexmock(),
  570. remote_path=flexmock(),
  571. borgmatic_runtime_directory='/run/borgmatic',
  572. ) == ('/etc/path', '/etc/other')
  573. def test_collect_spot_check_source_paths_skips_directories():
  574. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  575. {'hook1': False, 'hook2': True}
  576. )
  577. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  578. flexmock()
  579. )
  580. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  581. flexmock()
  582. )
  583. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  584. [Pattern('foo'), Pattern('bar')]
  585. )
  586. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  587. dry_run=True,
  588. repository_path='repo',
  589. config=object,
  590. patterns=[Pattern('foo'), Pattern('bar')],
  591. local_borg_version=object,
  592. global_arguments=object,
  593. borgmatic_runtime_directory='/run/borgmatic',
  594. local_path=object,
  595. remote_path=object,
  596. list_files=True,
  597. stream_processes=True,
  598. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  599. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  600. flexmock()
  601. )
  602. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  603. flexmock(module.borgmatic.execute).should_receive(
  604. 'execute_command_and_capture_output'
  605. ).and_return(
  606. 'warning: stuff\n- /etc/path\n+ /etc/dir\n? /nope',
  607. )
  608. flexmock(module.os.path).should_receive('isfile').with_args('/etc/path').and_return(False)
  609. flexmock(module.os.path).should_receive('isfile').with_args('/etc/dir').and_return(False)
  610. assert (
  611. module.collect_spot_check_source_paths(
  612. repository={'path': 'repo'},
  613. config={'working_directory': '/'},
  614. local_borg_version=flexmock(),
  615. global_arguments=flexmock(),
  616. local_path=flexmock(),
  617. remote_path=flexmock(),
  618. borgmatic_runtime_directory='/run/borgmatic',
  619. )
  620. == ()
  621. )
  622. def test_collect_spot_check_archive_paths_excludes_directories_and_pipes():
  623. flexmock(module.borgmatic.config.paths).should_receive(
  624. 'get_borgmatic_source_directory'
  625. ).and_return('/home/user/.borgmatic')
  626. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  627. (
  628. 'f etc/path',
  629. 'p var/pipe',
  630. 'f etc/other',
  631. 'd etc/dir',
  632. )
  633. )
  634. assert module.collect_spot_check_archive_paths(
  635. repository={'path': 'repo'},
  636. archive='archive',
  637. config={},
  638. local_borg_version=flexmock(),
  639. global_arguments=flexmock(),
  640. local_path=flexmock(),
  641. remote_path=flexmock(),
  642. borgmatic_runtime_directory='/run/user/1001/borgmatic',
  643. ) == ('etc/path', 'etc/other')
  644. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory_as_stored_with_prefix_truncation():
  645. flexmock(module.borgmatic.config.paths).should_receive(
  646. 'get_borgmatic_source_directory'
  647. ).and_return('/root/.borgmatic')
  648. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  649. (
  650. 'f etc/path',
  651. 'f borgmatic/some/thing',
  652. )
  653. )
  654. assert module.collect_spot_check_archive_paths(
  655. repository={'path': 'repo'},
  656. archive='archive',
  657. config={},
  658. local_borg_version=flexmock(),
  659. global_arguments=flexmock(),
  660. local_path=flexmock(),
  661. remote_path=flexmock(),
  662. borgmatic_runtime_directory='/run/user/0/borgmatic',
  663. ) == ('etc/path',)
  664. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory():
  665. flexmock(module.borgmatic.config.paths).should_receive(
  666. 'get_borgmatic_source_directory'
  667. ).and_return('/root/.borgmatic')
  668. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  669. (
  670. 'f etc/path',
  671. 'f root/.borgmatic/some/thing',
  672. )
  673. )
  674. assert module.collect_spot_check_archive_paths(
  675. repository={'path': 'repo'},
  676. archive='archive',
  677. config={},
  678. local_borg_version=flexmock(),
  679. global_arguments=flexmock(),
  680. local_path=flexmock(),
  681. remote_path=flexmock(),
  682. borgmatic_runtime_directory='/run/user/0/borgmatic',
  683. ) == ('etc/path',)
  684. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory():
  685. flexmock(module.borgmatic.config.paths).should_receive(
  686. 'get_borgmatic_source_directory'
  687. ).and_return('/root.borgmatic')
  688. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  689. (
  690. 'f etc/path',
  691. 'f run/user/0/borgmatic/some/thing',
  692. )
  693. )
  694. assert module.collect_spot_check_archive_paths(
  695. repository={'path': 'repo'},
  696. archive='archive',
  697. config={},
  698. local_borg_version=flexmock(),
  699. global_arguments=flexmock(),
  700. local_path=flexmock(),
  701. remote_path=flexmock(),
  702. borgmatic_runtime_directory='/run/user/0/borgmatic',
  703. ) == ('etc/path',)
  704. def test_collect_spot_check_source_paths_uses_working_directory():
  705. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  706. {'hook1': False, 'hook2': True}
  707. )
  708. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  709. flexmock()
  710. )
  711. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  712. flexmock()
  713. )
  714. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  715. [Pattern('foo'), Pattern('bar')]
  716. )
  717. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  718. dry_run=True,
  719. repository_path='repo',
  720. config=object,
  721. patterns=[Pattern('foo'), Pattern('bar')],
  722. local_borg_version=object,
  723. global_arguments=object,
  724. borgmatic_runtime_directory='/run/borgmatic',
  725. local_path=object,
  726. remote_path=object,
  727. list_files=True,
  728. stream_processes=True,
  729. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  730. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  731. flexmock()
  732. )
  733. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  734. '/working/dir'
  735. )
  736. flexmock(module.borgmatic.execute).should_receive(
  737. 'execute_command_and_capture_output'
  738. ).and_return(
  739. 'warning: stuff\n- foo\n+ bar\n? /nope',
  740. )
  741. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/foo').and_return(True)
  742. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/bar').and_return(True)
  743. assert module.collect_spot_check_source_paths(
  744. repository={'path': 'repo'},
  745. config={'working_directory': '/working/dir'},
  746. local_borg_version=flexmock(),
  747. global_arguments=flexmock(),
  748. local_path=flexmock(),
  749. remote_path=flexmock(),
  750. borgmatic_runtime_directory='/run/borgmatic',
  751. ) == ('foo', 'bar')
  752. def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
  753. flexmock(module.random).should_receive('sample').replace_with(
  754. lambda population, count: population[:count]
  755. )
  756. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  757. None,
  758. )
  759. flexmock(module.os.path).should_receive('exists').and_return(True)
  760. flexmock(module.borgmatic.execute).should_receive(
  761. 'execute_command_and_capture_output'
  762. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  763. 'hash1 /foo\nhash2 /bar'
  764. )
  765. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  766. ['hash1 foo', 'nothash2 bar']
  767. )
  768. assert module.compare_spot_check_hashes(
  769. repository={'path': 'repo'},
  770. archive='archive',
  771. config={
  772. 'checks': [
  773. {
  774. 'name': 'archives',
  775. 'frequency': '2 weeks',
  776. },
  777. {
  778. 'name': 'spot',
  779. 'data_sample_percentage': 50,
  780. },
  781. ]
  782. },
  783. local_borg_version=flexmock(),
  784. global_arguments=flexmock(),
  785. local_path=flexmock(),
  786. remote_path=flexmock(),
  787. source_paths=('/foo', '/bar', '/baz', '/quux'),
  788. ) == ('/bar',)
  789. def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes():
  790. flexmock(module.random).should_receive('sample').replace_with(
  791. lambda population, count: population[:count]
  792. )
  793. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  794. None,
  795. )
  796. flexmock(module.os.path).should_receive('exists').and_return(True)
  797. flexmock(module.borgmatic.execute).should_receive(
  798. 'execute_command_and_capture_output'
  799. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return(
  800. 'hash1 foo\nhash2 bar'
  801. )
  802. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  803. ['hash1 foo', 'nothash2 bar']
  804. )
  805. assert module.compare_spot_check_hashes(
  806. repository={'path': 'repo'},
  807. archive='archive',
  808. config={
  809. 'checks': [
  810. {
  811. 'name': 'archives',
  812. 'frequency': '2 weeks',
  813. },
  814. {
  815. 'name': 'spot',
  816. 'data_sample_percentage': 50,
  817. },
  818. ]
  819. },
  820. local_borg_version=flexmock(),
  821. global_arguments=flexmock(),
  822. local_path=flexmock(),
  823. remote_path=flexmock(),
  824. source_paths=('foo', 'bar', 'baz', 'quux'),
  825. ) == ('bar',)
  826. def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
  827. flexmock(module.random).should_receive('sample').replace_with(
  828. lambda population, count: population[:count]
  829. )
  830. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  831. None,
  832. )
  833. flexmock(module.os.path).should_receive('exists').and_return(True)
  834. flexmock(module.borgmatic.execute).should_receive(
  835. 'execute_command_and_capture_output'
  836. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  837. 'hash1 /foo\nhash2 /bar'
  838. )
  839. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  840. ['nothash1 foo', 'nothash2 bar']
  841. )
  842. assert module.compare_spot_check_hashes(
  843. repository={'path': 'repo'},
  844. archive='archive',
  845. config={
  846. 'checks': [
  847. {
  848. 'name': 'archives',
  849. 'frequency': '2 weeks',
  850. },
  851. {
  852. 'name': 'spot',
  853. 'data_sample_percentage': 1000,
  854. },
  855. ]
  856. },
  857. local_borg_version=flexmock(),
  858. global_arguments=flexmock(),
  859. local_path=flexmock(),
  860. remote_path=flexmock(),
  861. source_paths=('/foo', '/bar'),
  862. ) == ('/foo', '/bar')
  863. def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
  864. flexmock(module.random).should_receive('sample').replace_with(
  865. lambda population, count: population[:count]
  866. )
  867. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  868. None,
  869. )
  870. flexmock(module.os.path).should_receive('exists').and_return(True)
  871. flexmock(module.borgmatic.execute).should_receive(
  872. 'execute_command_and_capture_output'
  873. ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  874. 'hash1 /foo\nhash2 /bar'
  875. )
  876. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  877. ['hash1 foo', 'nothash2 bar']
  878. )
  879. assert module.compare_spot_check_hashes(
  880. repository={'path': 'repo'},
  881. archive='archive',
  882. config={
  883. 'checks': [
  884. {
  885. 'name': 'spot',
  886. 'data_sample_percentage': 50,
  887. 'xxh64sum_command': '/usr/local/bin/xxh64sum',
  888. },
  889. ]
  890. },
  891. local_borg_version=flexmock(),
  892. global_arguments=flexmock(),
  893. local_path=flexmock(),
  894. remote_path=flexmock(),
  895. source_paths=('/foo', '/bar', '/baz', '/quux'),
  896. ) == ('/bar',)
  897. def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_matching():
  898. flexmock(module.random).should_receive('sample').replace_with(
  899. lambda population, count: population[:count]
  900. )
  901. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  902. None,
  903. )
  904. flexmock(module.os.path).should_receive('exists').and_return(True)
  905. flexmock(module.borgmatic.execute).should_receive(
  906. 'execute_command_and_capture_output'
  907. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  908. 'hash1 /foo\nhash2 /bar'
  909. )
  910. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  911. ['hash1 foo']
  912. )
  913. assert module.compare_spot_check_hashes(
  914. repository={'path': 'repo'},
  915. archive='archive',
  916. config={
  917. 'checks': [
  918. {
  919. 'name': 'spot',
  920. 'data_sample_percentage': 50,
  921. },
  922. ]
  923. },
  924. local_borg_version=flexmock(),
  925. global_arguments=flexmock(),
  926. local_path=flexmock(),
  927. remote_path=flexmock(),
  928. source_paths=('/foo', '/bar', '/baz', '/quux'),
  929. ) == ('/bar',)
  930. def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching():
  931. flexmock(module.random).should_receive('sample').replace_with(
  932. lambda population, count: population[:count]
  933. )
  934. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  935. None,
  936. )
  937. flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
  938. flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False)
  939. flexmock(module.borgmatic.execute).should_receive(
  940. 'execute_command_and_capture_output'
  941. ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
  942. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  943. ['hash1 foo', 'hash2 bar']
  944. )
  945. assert module.compare_spot_check_hashes(
  946. repository={'path': 'repo'},
  947. archive='archive',
  948. config={
  949. 'checks': [
  950. {
  951. 'name': 'spot',
  952. 'data_sample_percentage': 50,
  953. },
  954. ]
  955. },
  956. local_borg_version=flexmock(),
  957. global_arguments=flexmock(),
  958. local_path=flexmock(),
  959. remote_path=flexmock(),
  960. source_paths=('/foo', '/bar', '/baz', '/quux'),
  961. ) == ('/bar',)
  962. def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in_chunks():
  963. flexmock(module).SAMPLE_PATHS_SUBSET_COUNT = 2
  964. flexmock(module.random).should_receive('sample').replace_with(
  965. lambda population, count: population[:count]
  966. )
  967. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  968. None,
  969. )
  970. flexmock(module.os.path).should_receive('exists').and_return(True)
  971. flexmock(module.borgmatic.execute).should_receive(
  972. 'execute_command_and_capture_output'
  973. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  974. 'hash1 /foo\nhash2 /bar'
  975. )
  976. flexmock(module.borgmatic.execute).should_receive(
  977. 'execute_command_and_capture_output'
  978. ).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_return(
  979. 'hash3 /baz\nhash4 /quux'
  980. )
  981. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  982. ['hash1 foo', 'hash2 bar']
  983. ).and_return(['hash3 baz', 'nothash4 quux'])
  984. assert module.compare_spot_check_hashes(
  985. repository={'path': 'repo'},
  986. archive='archive',
  987. config={
  988. 'checks': [
  989. {
  990. 'name': 'archives',
  991. 'frequency': '2 weeks',
  992. },
  993. {
  994. 'name': 'spot',
  995. 'data_sample_percentage': 100,
  996. },
  997. ]
  998. },
  999. local_borg_version=flexmock(),
  1000. global_arguments=flexmock(),
  1001. local_path=flexmock(),
  1002. remote_path=flexmock(),
  1003. source_paths=('/foo', '/bar', '/baz', '/quux'),
  1004. ) == ('/quux',)
  1005. def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths():
  1006. flexmock(module.random).should_receive('sample').replace_with(
  1007. lambda population, count: population[:count]
  1008. )
  1009. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  1010. '/working/dir',
  1011. )
  1012. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True)
  1013. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True)
  1014. flexmock(module.borgmatic.execute).should_receive(
  1015. 'execute_command_and_capture_output'
  1016. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return(
  1017. 'hash1 foo\nhash2 bar'
  1018. )
  1019. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  1020. ['hash1 foo', 'nothash2 bar']
  1021. )
  1022. assert module.compare_spot_check_hashes(
  1023. repository={'path': 'repo'},
  1024. archive='archive',
  1025. config={
  1026. 'checks': [
  1027. {
  1028. 'name': 'archives',
  1029. 'frequency': '2 weeks',
  1030. },
  1031. {
  1032. 'name': 'spot',
  1033. 'data_sample_percentage': 50,
  1034. },
  1035. ],
  1036. 'working_directory': '/working/dir',
  1037. },
  1038. local_borg_version=flexmock(),
  1039. global_arguments=flexmock(),
  1040. local_path=flexmock(),
  1041. remote_path=flexmock(),
  1042. source_paths=('foo', 'bar', 'baz', 'quux'),
  1043. ) == ('bar',)
  1044. def test_spot_check_without_spot_configuration_errors():
  1045. with pytest.raises(ValueError):
  1046. module.spot_check(
  1047. repository={'path': 'repo'},
  1048. config={
  1049. 'checks': [
  1050. {
  1051. 'name': 'archives',
  1052. },
  1053. ]
  1054. },
  1055. local_borg_version=flexmock(),
  1056. global_arguments=flexmock(),
  1057. local_path=flexmock(),
  1058. remote_path=flexmock(),
  1059. borgmatic_runtime_directory='/run/borgmatic',
  1060. )
  1061. def test_spot_check_without_any_configuration_errors():
  1062. with pytest.raises(ValueError):
  1063. module.spot_check(
  1064. repository={'path': 'repo'},
  1065. config={},
  1066. local_borg_version=flexmock(),
  1067. global_arguments=flexmock(),
  1068. local_path=flexmock(),
  1069. remote_path=flexmock(),
  1070. borgmatic_runtime_directory='/run/borgmatic',
  1071. )
  1072. def test_spot_check_data_tolerance_percentage_greater_than_data_sample_percentage_errors():
  1073. with pytest.raises(ValueError):
  1074. module.spot_check(
  1075. repository={'path': 'repo'},
  1076. config={
  1077. 'checks': [
  1078. {
  1079. 'name': 'spot',
  1080. 'data_tolerance_percentage': 7,
  1081. 'data_sample_percentage': 5,
  1082. },
  1083. ]
  1084. },
  1085. local_borg_version=flexmock(),
  1086. global_arguments=flexmock(),
  1087. local_path=flexmock(),
  1088. remote_path=flexmock(),
  1089. borgmatic_runtime_directory='/run/borgmatic',
  1090. )
  1091. def test_spot_check_with_count_delta_greater_than_count_tolerance_percentage_errors():
  1092. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1093. ('/foo', '/bar', '/baz', '/quux')
  1094. )
  1095. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1096. 'archive'
  1097. )
  1098. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(
  1099. ('/foo', '/bar')
  1100. ).once()
  1101. with pytest.raises(ValueError):
  1102. module.spot_check(
  1103. repository={'path': 'repo'},
  1104. config={
  1105. 'checks': [
  1106. {
  1107. 'name': 'spot',
  1108. 'count_tolerance_percentage': 1,
  1109. 'data_tolerance_percentage': 4,
  1110. 'data_sample_percentage': 5,
  1111. },
  1112. ]
  1113. },
  1114. local_borg_version=flexmock(),
  1115. global_arguments=flexmock(),
  1116. local_path=flexmock(),
  1117. remote_path=flexmock(),
  1118. borgmatic_runtime_directory='/run/borgmatic',
  1119. )
  1120. def test_spot_check_with_failing_percentage_greater_than_data_tolerance_percentage_errors():
  1121. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1122. ('/foo', '/bar', '/baz', '/quux')
  1123. )
  1124. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1125. 'archive'
  1126. )
  1127. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1128. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1129. ('/bar', '/baz', '/quux')
  1130. ).once()
  1131. with pytest.raises(ValueError):
  1132. module.spot_check(
  1133. repository={'path': 'repo'},
  1134. config={
  1135. 'checks': [
  1136. {
  1137. 'name': 'spot',
  1138. 'count_tolerance_percentage': 55,
  1139. 'data_tolerance_percentage': 4,
  1140. 'data_sample_percentage': 5,
  1141. },
  1142. ]
  1143. },
  1144. local_borg_version=flexmock(),
  1145. global_arguments=flexmock(),
  1146. local_path=flexmock(),
  1147. remote_path=flexmock(),
  1148. borgmatic_runtime_directory='/run/borgmatic',
  1149. )
  1150. def test_spot_check_with_high_enough_tolerances_does_not_raise():
  1151. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1152. ('/foo', '/bar', '/baz', '/quux')
  1153. )
  1154. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1155. 'archive'
  1156. )
  1157. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1158. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1159. ('/bar', '/baz', '/quux')
  1160. ).once()
  1161. module.spot_check(
  1162. repository={'path': 'repo'},
  1163. config={
  1164. 'checks': [
  1165. {
  1166. 'name': 'spot',
  1167. 'count_tolerance_percentage': 55,
  1168. 'data_tolerance_percentage': 80,
  1169. 'data_sample_percentage': 80,
  1170. },
  1171. ]
  1172. },
  1173. local_borg_version=flexmock(),
  1174. global_arguments=flexmock(),
  1175. local_path=flexmock(),
  1176. remote_path=flexmock(),
  1177. borgmatic_runtime_directory='/run/borgmatic',
  1178. )
  1179. def test_spot_check_without_any_source_paths_errors():
  1180. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(())
  1181. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1182. 'archive'
  1183. )
  1184. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1185. flexmock(module).should_receive('compare_spot_check_hashes').never()
  1186. with pytest.raises(ValueError):
  1187. module.spot_check(
  1188. repository={'path': 'repo'},
  1189. config={
  1190. 'checks': [
  1191. {
  1192. 'name': 'spot',
  1193. 'count_tolerance_percentage': 10,
  1194. 'data_tolerance_percentage': 40,
  1195. 'data_sample_percentage': 50,
  1196. },
  1197. ]
  1198. },
  1199. local_borg_version=flexmock(),
  1200. global_arguments=flexmock(),
  1201. local_path=flexmock(),
  1202. remote_path=flexmock(),
  1203. borgmatic_runtime_directory='/run/borgmatic',
  1204. )
  1205. def test_run_check_checks_archives_for_configured_repository():
  1206. flexmock(module.logger).answer = lambda message: None
  1207. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1208. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1209. flexmock(module).should_receive('upgrade_check_times')
  1210. flexmock(module).should_receive('parse_checks')
  1211. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1212. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1213. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1214. {'repository', 'archives'}
  1215. )
  1216. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1217. flexmock(module).should_receive('make_check_time_path')
  1218. flexmock(module).should_receive('write_check_time')
  1219. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1220. check_arguments = flexmock(
  1221. repository=None,
  1222. progress=flexmock(),
  1223. repair=flexmock(),
  1224. only_checks=flexmock(),
  1225. force=flexmock(),
  1226. )
  1227. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1228. module.run_check(
  1229. config_filename='test.yaml',
  1230. repository={'path': 'repo'},
  1231. config={'repositories': ['repo']},
  1232. local_borg_version=None,
  1233. check_arguments=check_arguments,
  1234. global_arguments=global_arguments,
  1235. local_path=None,
  1236. remote_path=None,
  1237. )
  1238. def test_run_check_runs_configured_extract_check():
  1239. flexmock(module.logger).answer = lambda message: None
  1240. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1241. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1242. flexmock(module).should_receive('upgrade_check_times')
  1243. flexmock(module).should_receive('parse_checks')
  1244. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1245. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1246. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'extract'})
  1247. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1248. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').once()
  1249. flexmock(module).should_receive('make_check_time_path')
  1250. flexmock(module).should_receive('write_check_time')
  1251. check_arguments = flexmock(
  1252. repository=None,
  1253. progress=flexmock(),
  1254. repair=flexmock(),
  1255. only_checks=flexmock(),
  1256. force=flexmock(),
  1257. )
  1258. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1259. module.run_check(
  1260. config_filename='test.yaml',
  1261. repository={'path': 'repo'},
  1262. config={'repositories': ['repo']},
  1263. local_borg_version=None,
  1264. check_arguments=check_arguments,
  1265. global_arguments=global_arguments,
  1266. local_path=None,
  1267. remote_path=None,
  1268. )
  1269. def test_run_check_runs_configured_spot_check():
  1270. flexmock(module.logger).answer = lambda message: None
  1271. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1272. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1273. flexmock(module).should_receive('upgrade_check_times')
  1274. flexmock(module).should_receive('parse_checks')
  1275. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1276. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1277. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'spot'})
  1278. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1279. flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
  1280. flexmock()
  1281. )
  1282. flexmock(module.borgmatic.actions.check).should_receive('spot_check').once()
  1283. flexmock(module).should_receive('make_check_time_path')
  1284. flexmock(module).should_receive('write_check_time')
  1285. check_arguments = flexmock(
  1286. repository=None,
  1287. progress=flexmock(),
  1288. repair=flexmock(),
  1289. only_checks=flexmock(),
  1290. force=flexmock(),
  1291. )
  1292. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1293. module.run_check(
  1294. config_filename='test.yaml',
  1295. repository={'path': 'repo'},
  1296. config={'repositories': ['repo']},
  1297. local_borg_version=None,
  1298. check_arguments=check_arguments,
  1299. global_arguments=global_arguments,
  1300. local_path=None,
  1301. remote_path=None,
  1302. )
  1303. def test_run_check_without_checks_runs_nothing_except_hooks():
  1304. flexmock(module.logger).answer = lambda message: None
  1305. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1306. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1307. flexmock(module).should_receive('upgrade_check_times')
  1308. flexmock(module).should_receive('parse_checks')
  1309. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1310. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1311. flexmock(module).should_receive('filter_checks_on_frequency').and_return({})
  1312. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1313. flexmock(module).should_receive('make_check_time_path')
  1314. flexmock(module).should_receive('write_check_time').never()
  1315. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1316. check_arguments = flexmock(
  1317. repository=None,
  1318. progress=flexmock(),
  1319. repair=flexmock(),
  1320. only_checks=flexmock(),
  1321. force=flexmock(),
  1322. )
  1323. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1324. module.run_check(
  1325. config_filename='test.yaml',
  1326. repository={'path': 'repo'},
  1327. config={'repositories': ['repo']},
  1328. local_borg_version=None,
  1329. check_arguments=check_arguments,
  1330. global_arguments=global_arguments,
  1331. local_path=None,
  1332. remote_path=None,
  1333. )
  1334. def test_run_check_checks_archives_in_selected_repository():
  1335. flexmock(module.logger).answer = lambda message: None
  1336. flexmock(module.borgmatic.config.validate).should_receive(
  1337. 'repositories_match'
  1338. ).once().and_return(True)
  1339. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1340. flexmock(module).should_receive('upgrade_check_times')
  1341. flexmock(module).should_receive('parse_checks')
  1342. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1343. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1344. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1345. {'repository', 'archives'}
  1346. )
  1347. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1348. flexmock(module).should_receive('make_check_time_path')
  1349. flexmock(module).should_receive('write_check_time')
  1350. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1351. check_arguments = flexmock(
  1352. repository=flexmock(),
  1353. progress=flexmock(),
  1354. repair=flexmock(),
  1355. only_checks=flexmock(),
  1356. force=flexmock(),
  1357. )
  1358. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1359. module.run_check(
  1360. config_filename='test.yaml',
  1361. repository={'path': 'repo'},
  1362. config={'repositories': ['repo']},
  1363. local_borg_version=None,
  1364. check_arguments=check_arguments,
  1365. global_arguments=global_arguments,
  1366. local_path=None,
  1367. remote_path=None,
  1368. )
  1369. def test_run_check_bails_if_repository_does_not_match():
  1370. flexmock(module.logger).answer = lambda message: None
  1371. flexmock(module.borgmatic.config.validate).should_receive(
  1372. 'repositories_match'
  1373. ).once().and_return(False)
  1374. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1375. check_arguments = flexmock(
  1376. repository=flexmock(),
  1377. progress=flexmock(),
  1378. repair=flexmock(),
  1379. only_checks=flexmock(),
  1380. force=flexmock(),
  1381. )
  1382. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1383. module.run_check(
  1384. config_filename='test.yaml',
  1385. repository={'path': 'repo'},
  1386. config={'repositories': ['repo']},
  1387. local_borg_version=None,
  1388. check_arguments=check_arguments,
  1389. global_arguments=global_arguments,
  1390. local_path=None,
  1391. remote_path=None,
  1392. )