test_check.py 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606
  1. import pytest
  2. from flexmock import flexmock
  3. from borgmatic.actions import check as module
  4. from borgmatic.borg.pattern import Pattern
  5. def test_parse_checks_returns_them_as_tuple():
  6. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'bar'}]})
  7. assert checks == ('foo', 'bar')
  8. def test_parse_checks_with_missing_value_returns_defaults():
  9. checks = module.parse_checks({})
  10. assert checks == ('repository', 'archives')
  11. def test_parse_checks_with_empty_list_returns_defaults():
  12. checks = module.parse_checks({'checks': []})
  13. assert checks == ('repository', 'archives')
  14. def test_parse_checks_with_none_value_returns_defaults():
  15. checks = module.parse_checks({'checks': None})
  16. assert checks == ('repository', 'archives')
  17. def test_parse_checks_with_disabled_returns_no_checks():
  18. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'disabled'}]})
  19. assert checks == ()
  20. def test_parse_checks_prefers_override_checks_to_configured_checks():
  21. checks = module.parse_checks(
  22. {'checks': [{'name': 'archives'}]}, only_checks=['repository', 'extract']
  23. )
  24. assert checks == ('repository', 'extract')
  25. @pytest.mark.parametrize(
  26. 'frequency,expected_result',
  27. (
  28. (None, None),
  29. ('always', None),
  30. ('1 hour', module.datetime.timedelta(hours=1)),
  31. ('2 hours', module.datetime.timedelta(hours=2)),
  32. ('1 day', module.datetime.timedelta(days=1)),
  33. ('2 days', module.datetime.timedelta(days=2)),
  34. ('1 week', module.datetime.timedelta(weeks=1)),
  35. ('2 weeks', module.datetime.timedelta(weeks=2)),
  36. ('1 month', module.datetime.timedelta(days=30)),
  37. ('2 months', module.datetime.timedelta(days=60)),
  38. ('1 year', module.datetime.timedelta(days=365)),
  39. ('2 years', module.datetime.timedelta(days=365 * 2)),
  40. ),
  41. )
  42. def test_parse_frequency_parses_into_timedeltas(frequency, expected_result):
  43. assert module.parse_frequency(frequency) == expected_result
  44. @pytest.mark.parametrize(
  45. 'frequency',
  46. (
  47. 'sometime',
  48. 'x days',
  49. '3 decades',
  50. ),
  51. )
  52. def test_parse_frequency_raises_on_parse_error(frequency):
  53. with pytest.raises(ValueError):
  54. module.parse_frequency(frequency)
  55. def test_filter_checks_on_frequency_without_config_uses_default_checks():
  56. flexmock(module).should_receive('parse_frequency').and_return(
  57. module.datetime.timedelta(weeks=4)
  58. )
  59. flexmock(module).should_receive('make_check_time_path')
  60. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  61. assert module.filter_checks_on_frequency(
  62. config={},
  63. borg_repository_id='repo',
  64. checks=('repository', 'archives'),
  65. force=False,
  66. archives_check_id='1234',
  67. ) == ('repository', 'archives')
  68. def test_filter_checks_on_frequency_retains_unconfigured_check():
  69. assert module.filter_checks_on_frequency(
  70. config={},
  71. borg_repository_id='repo',
  72. checks=('data',),
  73. force=False,
  74. ) == ('data',)
  75. def test_filter_checks_on_frequency_retains_check_without_frequency():
  76. flexmock(module).should_receive('parse_frequency').and_return(None)
  77. assert module.filter_checks_on_frequency(
  78. config={'checks': [{'name': 'archives'}]},
  79. borg_repository_id='repo',
  80. checks=('archives',),
  81. force=False,
  82. archives_check_id='1234',
  83. ) == ('archives',)
  84. def test_filter_checks_on_frequency_retains_check_with_empty_only_run_on():
  85. flexmock(module).should_receive('parse_frequency').and_return(None)
  86. assert module.filter_checks_on_frequency(
  87. config={'checks': [{'name': 'archives', 'only_run_on': []}]},
  88. borg_repository_id='repo',
  89. checks=('archives',),
  90. force=False,
  91. archives_check_id='1234',
  92. datetime_now=flexmock(weekday=lambda: 0),
  93. ) == ('archives',)
  94. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today():
  95. flexmock(module).should_receive('parse_frequency').and_return(None)
  96. assert module.filter_checks_on_frequency(
  97. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[0]]}]},
  98. borg_repository_id='repo',
  99. checks=('archives',),
  100. force=False,
  101. archives_check_id='1234',
  102. datetime_now=flexmock(weekday=lambda: 0),
  103. ) == ('archives',)
  104. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekday_value():
  105. flexmock(module).should_receive('parse_frequency').and_return(None)
  106. assert module.filter_checks_on_frequency(
  107. config={'checks': [{'name': 'archives', 'only_run_on': ['weekday']}]},
  108. borg_repository_id='repo',
  109. checks=('archives',),
  110. force=False,
  111. archives_check_id='1234',
  112. datetime_now=flexmock(weekday=lambda: 0),
  113. ) == ('archives',)
  114. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekend_value():
  115. flexmock(module).should_receive('parse_frequency').and_return(None)
  116. assert module.filter_checks_on_frequency(
  117. config={'checks': [{'name': 'archives', 'only_run_on': ['weekend']}]},
  118. borg_repository_id='repo',
  119. checks=('archives',),
  120. force=False,
  121. archives_check_id='1234',
  122. datetime_now=flexmock(weekday=lambda: 6),
  123. ) == ('archives',)
  124. def test_filter_checks_on_frequency_skips_check_with_only_run_on_not_matching_today():
  125. flexmock(module).should_receive('parse_frequency').and_return(None)
  126. assert (
  127. module.filter_checks_on_frequency(
  128. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[5]]}]},
  129. borg_repository_id='repo',
  130. checks=('archives',),
  131. force=False,
  132. archives_check_id='1234',
  133. datetime_now=flexmock(weekday=lambda: 0),
  134. )
  135. == ()
  136. )
  137. def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
  138. flexmock(module).should_receive('parse_frequency').and_return(
  139. module.datetime.timedelta(hours=1)
  140. )
  141. flexmock(module).should_receive('make_check_time_path')
  142. flexmock(module).should_receive('probe_for_check_time').and_return(
  143. module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1)
  144. )
  145. assert module.filter_checks_on_frequency(
  146. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  147. borg_repository_id='repo',
  148. checks=('archives',),
  149. force=False,
  150. archives_check_id='1234',
  151. ) == ('archives',)
  152. def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file():
  153. flexmock(module).should_receive('parse_frequency').and_return(
  154. module.datetime.timedelta(hours=1)
  155. )
  156. flexmock(module).should_receive('make_check_time_path')
  157. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  158. assert module.filter_checks_on_frequency(
  159. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  160. borg_repository_id='repo',
  161. checks=('archives',),
  162. force=False,
  163. archives_check_id='1234',
  164. ) == ('archives',)
  165. def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
  166. flexmock(module).should_receive('parse_frequency').and_return(
  167. module.datetime.timedelta(hours=1)
  168. )
  169. flexmock(module).should_receive('make_check_time_path')
  170. flexmock(module).should_receive('probe_for_check_time').and_return(
  171. module.datetime.datetime.now()
  172. )
  173. assert (
  174. module.filter_checks_on_frequency(
  175. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  176. borg_repository_id='repo',
  177. checks=('archives',),
  178. force=False,
  179. archives_check_id='1234',
  180. )
  181. == ()
  182. )
  183. def test_filter_checks_on_frequency_retains_check_with_unelapsed_frequency_and_force():
  184. assert module.filter_checks_on_frequency(
  185. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  186. borg_repository_id='repo',
  187. checks=('archives',),
  188. force=True,
  189. archives_check_id='1234',
  190. ) == ('archives',)
  191. def test_filter_checks_on_frequency_passes_through_empty_checks():
  192. assert (
  193. module.filter_checks_on_frequency(
  194. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  195. borg_repository_id='repo',
  196. checks=(),
  197. force=False,
  198. archives_check_id='1234',
  199. )
  200. == ()
  201. )
  202. def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise():
  203. assert module.make_archives_check_id(('--match-archives', 'sh:foo-*'))
  204. def test_make_archives_check_id_with_empty_flags_returns_none():
  205. assert module.make_archives_check_id(()) is None
  206. def test_make_check_time_path_with_borgmatic_source_directory_includes_it():
  207. flexmock(module.borgmatic.config.paths).should_receive(
  208. 'get_borgmatic_state_directory'
  209. ).and_return(
  210. '/home/user/.local/state/borgmatic',
  211. )
  212. assert (
  213. module.make_check_time_path({}, '1234', 'archives', '5678')
  214. == '/home/user/.local/state/borgmatic/checks/1234/archives/5678'
  215. )
  216. def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all():
  217. flexmock(module.borgmatic.config.paths).should_receive(
  218. 'get_borgmatic_state_directory'
  219. ).and_return(
  220. '/home/user/.local/state/borgmatic',
  221. )
  222. assert (
  223. module.make_check_time_path(
  224. {},
  225. '1234',
  226. 'archives',
  227. )
  228. == '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  229. )
  230. def test_make_check_time_path_with_repositories_check_ignores_archives_check_id():
  231. flexmock(module.borgmatic.config.paths).should_receive(
  232. 'get_borgmatic_state_directory'
  233. ).and_return(
  234. '/home/user/.local/state/borgmatic',
  235. )
  236. assert (
  237. module.make_check_time_path({}, '1234', 'repository', '5678')
  238. == '/home/user/.local/state/borgmatic/checks/1234/repository'
  239. )
  240. def test_read_check_time_does_not_raise():
  241. flexmock(module.os).should_receive('stat').and_return(flexmock(st_mtime=123))
  242. assert module.read_check_time('/path')
  243. def test_read_check_time_on_missing_file_does_not_raise():
  244. flexmock(module.os).should_receive('stat').and_raise(FileNotFoundError)
  245. assert module.read_check_time('/path') is None
  246. def test_probe_for_check_time_uses_maximum_of_multiple_check_times():
  247. flexmock(module).should_receive('make_check_time_path').and_return(
  248. '~/.borgmatic/checks/1234/archives/5678'
  249. ).and_return('~/.borgmatic/checks/1234/archives/all')
  250. flexmock(module).should_receive('read_check_time').and_return(1).and_return(2)
  251. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  252. def test_probe_for_check_time_deduplicates_identical_check_time_paths():
  253. flexmock(module).should_receive('make_check_time_path').and_return(
  254. '~/.borgmatic/checks/1234/archives/5678'
  255. ).and_return('~/.borgmatic/checks/1234/archives/5678')
  256. flexmock(module).should_receive('read_check_time').and_return(1).once()
  257. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  258. def test_probe_for_check_time_skips_none_check_time():
  259. flexmock(module).should_receive('make_check_time_path').and_return(
  260. '~/.borgmatic/checks/1234/archives/5678'
  261. ).and_return('~/.borgmatic/checks/1234/archives/all')
  262. flexmock(module).should_receive('read_check_time').and_return(None).and_return(2)
  263. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  264. def test_probe_for_check_time_uses_single_check_time():
  265. flexmock(module).should_receive('make_check_time_path').and_return(
  266. '~/.borgmatic/checks/1234/archives/5678'
  267. ).and_return('~/.borgmatic/checks/1234/archives/all')
  268. flexmock(module).should_receive('read_check_time').and_return(1).and_return(None)
  269. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  270. def test_probe_for_check_time_returns_none_when_no_check_time_found():
  271. flexmock(module).should_receive('make_check_time_path').and_return(
  272. '~/.borgmatic/checks/1234/archives/5678'
  273. ).and_return('~/.borgmatic/checks/1234/archives/all')
  274. flexmock(module).should_receive('read_check_time').and_return(None).and_return(None)
  275. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None
  276. def test_upgrade_check_times_moves_checks_from_borgmatic_source_directory_to_state_directory():
  277. flexmock(module.borgmatic.config.paths).should_receive(
  278. 'get_borgmatic_source_directory'
  279. ).and_return('/home/user/.borgmatic')
  280. flexmock(module.borgmatic.config.paths).should_receive(
  281. 'get_borgmatic_state_directory'
  282. ).and_return('/home/user/.local/state/borgmatic')
  283. flexmock(module.os.path).should_receive('exists').with_args(
  284. '/home/user/.borgmatic/checks'
  285. ).and_return(True)
  286. flexmock(module.os.path).should_receive('exists').with_args(
  287. '/home/user/.local/state/borgmatic/checks'
  288. ).and_return(False)
  289. flexmock(module.os).should_receive('makedirs')
  290. flexmock(module.shutil).should_receive('move').with_args(
  291. '/home/user/.borgmatic/checks', '/home/user/.local/state/borgmatic/checks'
  292. ).once()
  293. flexmock(module).should_receive('make_check_time_path').and_return(
  294. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  295. )
  296. flexmock(module.os.path).should_receive('isfile').and_return(False)
  297. flexmock(module.os).should_receive('mkdir').never()
  298. module.upgrade_check_times(flexmock(), flexmock())
  299. def test_upgrade_check_times_with_checks_already_in_borgmatic_state_directory_does_not_move_anything():
  300. flexmock(module.borgmatic.config.paths).should_receive(
  301. 'get_borgmatic_source_directory'
  302. ).and_return('/home/user/.borgmatic')
  303. flexmock(module.borgmatic.config.paths).should_receive(
  304. 'get_borgmatic_state_directory'
  305. ).and_return('/home/user/.local/state/borgmatic')
  306. flexmock(module.os.path).should_receive('exists').with_args(
  307. '/home/user/.borgmatic/checks'
  308. ).and_return(True)
  309. flexmock(module.os.path).should_receive('exists').with_args(
  310. '/home/user/.local/state/borgmatic/checks'
  311. ).and_return(True)
  312. flexmock(module.os).should_receive('makedirs').never()
  313. flexmock(module.shutil).should_receive('move').never()
  314. flexmock(module).should_receive('make_check_time_path').and_return(
  315. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  316. )
  317. flexmock(module.os.path).should_receive('isfile').and_return(False)
  318. flexmock(module.shutil).should_receive('move').never()
  319. flexmock(module.os).should_receive('mkdir').never()
  320. module.upgrade_check_times(flexmock(), flexmock())
  321. def test_upgrade_check_times_renames_old_check_paths_to_all():
  322. flexmock(module.borgmatic.config.paths).should_receive(
  323. 'get_borgmatic_source_directory'
  324. ).and_return('/home/user/.borgmatic')
  325. flexmock(module.borgmatic.config.paths).should_receive(
  326. 'get_borgmatic_state_directory'
  327. ).and_return('/home/user/.local/state/borgmatic')
  328. flexmock(module.os.path).should_receive('exists').and_return(False)
  329. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  330. flexmock(module).should_receive('make_check_time_path').with_args(
  331. object, object, 'archives', 'all'
  332. ).and_return(f'{base_path}/archives/all')
  333. flexmock(module).should_receive('make_check_time_path').with_args(
  334. object, object, 'data', 'all'
  335. ).and_return(f'{base_path}/data/all')
  336. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  337. True
  338. )
  339. flexmock(module.os.path).should_receive('isfile').with_args(
  340. f'{base_path}/archives.temp'
  341. ).and_return(False)
  342. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  343. False
  344. )
  345. flexmock(module.os.path).should_receive('isfile').with_args(
  346. f'{base_path}/data.temp'
  347. ).and_return(False)
  348. flexmock(module.shutil).should_receive('move').with_args(
  349. f'{base_path}/archives', f'{base_path}/archives.temp'
  350. ).once()
  351. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  352. flexmock(module.shutil).should_receive('move').with_args(
  353. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  354. ).once()
  355. module.upgrade_check_times(flexmock(), flexmock())
  356. def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded():
  357. flexmock(module.borgmatic.config.paths).should_receive(
  358. 'get_borgmatic_source_directory'
  359. ).and_return('/home/user/.borgmatic')
  360. flexmock(module.borgmatic.config.paths).should_receive(
  361. 'get_borgmatic_state_directory'
  362. ).and_return('/home/user/.local/state/borgmatic')
  363. flexmock(module.os.path).should_receive('exists').and_return(False)
  364. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  365. flexmock(module).should_receive('make_check_time_path').with_args(
  366. object, object, 'archives', 'all'
  367. ).and_return(f'{base_path}/archives/all')
  368. flexmock(module).should_receive('make_check_time_path').with_args(
  369. object, object, 'data', 'all'
  370. ).and_return(f'{base_path}/data/all')
  371. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  372. False
  373. )
  374. flexmock(module.os.path).should_receive('isfile').with_args(
  375. f'{base_path}/archives.temp'
  376. ).and_return(False)
  377. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  378. True
  379. )
  380. flexmock(module.shutil).should_receive('move').with_args(
  381. f'{base_path}/data', f'{base_path}/data.temp'
  382. ).once()
  383. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/data').once()
  384. flexmock(module.shutil).should_receive('move').with_args(
  385. f'{base_path}/data.temp', f'{base_path}/data/all'
  386. ).once()
  387. module.upgrade_check_times(flexmock(), flexmock())
  388. def test_upgrade_check_times_skips_already_upgraded_check_paths():
  389. flexmock(module.borgmatic.config.paths).should_receive(
  390. 'get_borgmatic_source_directory'
  391. ).and_return('/home/user/.borgmatic')
  392. flexmock(module.borgmatic.config.paths).should_receive(
  393. 'get_borgmatic_state_directory'
  394. ).and_return('/home/user/.local/state/borgmatic')
  395. flexmock(module.os.path).should_receive('exists').and_return(False)
  396. flexmock(module).should_receive('make_check_time_path').and_return(
  397. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  398. )
  399. flexmock(module.os.path).should_receive('isfile').and_return(False)
  400. flexmock(module.shutil).should_receive('move').never()
  401. flexmock(module.os).should_receive('mkdir').never()
  402. module.upgrade_check_times(flexmock(), flexmock())
  403. def test_upgrade_check_times_renames_stale_temporary_check_path():
  404. flexmock(module.borgmatic.config.paths).should_receive(
  405. 'get_borgmatic_source_directory'
  406. ).and_return('/home/user/.borgmatic')
  407. flexmock(module.borgmatic.config.paths).should_receive(
  408. 'get_borgmatic_state_directory'
  409. ).and_return('/home/user/.local/state/borgmatic')
  410. flexmock(module.os.path).should_receive('exists').and_return(False)
  411. base_path = '/home/borgmatic/.local/state/checks/1234'
  412. flexmock(module).should_receive('make_check_time_path').with_args(
  413. object, object, 'archives', 'all'
  414. ).and_return(f'{base_path}/archives/all')
  415. flexmock(module).should_receive('make_check_time_path').with_args(
  416. object, object, 'data', 'all'
  417. ).and_return(f'{base_path}/data/all')
  418. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  419. False
  420. )
  421. flexmock(module.os.path).should_receive('isfile').with_args(
  422. f'{base_path}/archives.temp'
  423. ).and_return(True)
  424. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  425. False
  426. )
  427. flexmock(module.os.path).should_receive('isfile').with_args(
  428. f'{base_path}/data.temp'
  429. ).and_return(False)
  430. flexmock(module.shutil).should_receive('move').with_args(
  431. f'{base_path}/archives', f'{base_path}/archives.temp'
  432. ).and_raise(FileNotFoundError)
  433. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  434. flexmock(module.shutil).should_receive('move').with_args(
  435. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  436. ).once()
  437. module.upgrade_check_times(flexmock(), flexmock())
  438. def test_collect_spot_check_source_paths_parses_borg_output():
  439. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  440. {'hook1': False, 'hook2': True}
  441. )
  442. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  443. flexmock()
  444. )
  445. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  446. flexmock()
  447. )
  448. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  449. [Pattern('foo'), Pattern('bar')]
  450. )
  451. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  452. dry_run=True,
  453. repository_path='repo',
  454. config=object,
  455. patterns=[Pattern('foo'), Pattern('bar')],
  456. local_borg_version=object,
  457. global_arguments=object,
  458. borgmatic_runtime_directory='/run/borgmatic',
  459. local_path=object,
  460. remote_path=object,
  461. list_files=True,
  462. stream_processes=True,
  463. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  464. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  465. flexmock()
  466. )
  467. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  468. flexmock(module.borgmatic.execute).should_receive(
  469. 'execute_command_and_capture_output'
  470. ).and_return(
  471. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  472. )
  473. flexmock(module.os.path).should_receive('isfile').and_return(True)
  474. assert module.collect_spot_check_source_paths(
  475. repository={'path': 'repo'},
  476. config={'working_directory': '/'},
  477. local_borg_version=flexmock(),
  478. global_arguments=flexmock(),
  479. local_path=flexmock(),
  480. remote_path=flexmock(),
  481. borgmatic_runtime_directory='/run/borgmatic',
  482. ) == ('/etc/path', '/etc/other')
  483. def test_collect_spot_check_source_paths_passes_through_stream_processes_false():
  484. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  485. {'hook1': False, 'hook2': False}
  486. )
  487. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  488. flexmock()
  489. )
  490. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  491. flexmock()
  492. )
  493. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  494. [Pattern('foo'), Pattern('bar')]
  495. )
  496. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  497. dry_run=True,
  498. repository_path='repo',
  499. config=object,
  500. patterns=[Pattern('foo'), Pattern('bar')],
  501. local_borg_version=object,
  502. global_arguments=object,
  503. borgmatic_runtime_directory='/run/borgmatic',
  504. local_path=object,
  505. remote_path=object,
  506. list_files=True,
  507. stream_processes=False,
  508. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  509. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  510. flexmock()
  511. )
  512. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  513. flexmock(module.borgmatic.execute).should_receive(
  514. 'execute_command_and_capture_output'
  515. ).and_return(
  516. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  517. )
  518. flexmock(module.os.path).should_receive('isfile').and_return(True)
  519. assert module.collect_spot_check_source_paths(
  520. repository={'path': 'repo'},
  521. config={'working_directory': '/'},
  522. local_borg_version=flexmock(),
  523. global_arguments=flexmock(),
  524. local_path=flexmock(),
  525. remote_path=flexmock(),
  526. borgmatic_runtime_directory='/run/borgmatic',
  527. ) == ('/etc/path', '/etc/other')
  528. def test_collect_spot_check_source_paths_without_working_directory_parses_borg_output():
  529. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  530. {'hook1': False, 'hook2': True}
  531. )
  532. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  533. flexmock()
  534. )
  535. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  536. flexmock()
  537. )
  538. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  539. [Pattern('foo'), Pattern('bar')]
  540. )
  541. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  542. dry_run=True,
  543. repository_path='repo',
  544. config=object,
  545. patterns=[Pattern('foo'), Pattern('bar')],
  546. local_borg_version=object,
  547. global_arguments=object,
  548. borgmatic_runtime_directory='/run/borgmatic',
  549. local_path=object,
  550. remote_path=object,
  551. list_files=True,
  552. stream_processes=True,
  553. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  554. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  555. flexmock()
  556. )
  557. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  558. flexmock(module.borgmatic.execute).should_receive(
  559. 'execute_command_and_capture_output'
  560. ).and_return(
  561. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  562. )
  563. flexmock(module.os.path).should_receive('isfile').and_return(True)
  564. assert module.collect_spot_check_source_paths(
  565. repository={'path': 'repo'},
  566. config={},
  567. local_borg_version=flexmock(),
  568. global_arguments=flexmock(),
  569. local_path=flexmock(),
  570. remote_path=flexmock(),
  571. borgmatic_runtime_directory='/run/borgmatic',
  572. ) == ('/etc/path', '/etc/other')
  573. def test_collect_spot_check_source_paths_skips_directories():
  574. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  575. {'hook1': False, 'hook2': True}
  576. )
  577. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  578. flexmock()
  579. )
  580. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  581. flexmock()
  582. )
  583. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  584. [Pattern('foo'), Pattern('bar')]
  585. )
  586. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  587. dry_run=True,
  588. repository_path='repo',
  589. config=object,
  590. patterns=[Pattern('foo'), Pattern('bar')],
  591. local_borg_version=object,
  592. global_arguments=object,
  593. borgmatic_runtime_directory='/run/borgmatic',
  594. local_path=object,
  595. remote_path=object,
  596. list_files=True,
  597. stream_processes=True,
  598. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  599. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  600. flexmock()
  601. )
  602. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  603. flexmock(module.borgmatic.execute).should_receive(
  604. 'execute_command_and_capture_output'
  605. ).and_return(
  606. 'warning: stuff\n- /etc/path\n+ /etc/dir\n? /nope',
  607. )
  608. flexmock(module.os.path).should_receive('isfile').with_args('/etc/path').and_return(False)
  609. flexmock(module.os.path).should_receive('isfile').with_args('/etc/dir').and_return(False)
  610. assert (
  611. module.collect_spot_check_source_paths(
  612. repository={'path': 'repo'},
  613. config={'working_directory': '/'},
  614. local_borg_version=flexmock(),
  615. global_arguments=flexmock(),
  616. local_path=flexmock(),
  617. remote_path=flexmock(),
  618. borgmatic_runtime_directory='/run/borgmatic',
  619. )
  620. == ()
  621. )
  622. def test_collect_spot_check_archive_paths_excludes_directories_and_pipes():
  623. flexmock(module.borgmatic.config.paths).should_receive(
  624. 'get_borgmatic_source_directory'
  625. ).and_return('/home/user/.borgmatic')
  626. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  627. (
  628. 'f etc/path',
  629. 'p var/pipe',
  630. 'f etc/other',
  631. 'd etc/dir',
  632. )
  633. )
  634. assert module.collect_spot_check_archive_paths(
  635. repository={'path': 'repo'},
  636. archive='archive',
  637. config={},
  638. local_borg_version=flexmock(),
  639. global_arguments=flexmock(),
  640. local_path=flexmock(),
  641. remote_path=flexmock(),
  642. borgmatic_runtime_directory='/run/user/1001/borgmatic',
  643. ) == ('etc/path', 'etc/other')
  644. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory_as_stored_with_prefix_truncation():
  645. flexmock(module.borgmatic.config.paths).should_receive(
  646. 'get_borgmatic_source_directory'
  647. ).and_return('/root/.borgmatic')
  648. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  649. (
  650. 'f etc/path',
  651. 'f borgmatic/some/thing',
  652. )
  653. )
  654. assert module.collect_spot_check_archive_paths(
  655. repository={'path': 'repo'},
  656. archive='archive',
  657. config={},
  658. local_borg_version=flexmock(),
  659. global_arguments=flexmock(),
  660. local_path=flexmock(),
  661. remote_path=flexmock(),
  662. borgmatic_runtime_directory='/run/user/0/borgmatic',
  663. ) == ('etc/path',)
  664. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory():
  665. flexmock(module.borgmatic.config.paths).should_receive(
  666. 'get_borgmatic_source_directory'
  667. ).and_return('/root/.borgmatic')
  668. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  669. (
  670. 'f etc/path',
  671. 'f root/.borgmatic/some/thing',
  672. )
  673. )
  674. assert module.collect_spot_check_archive_paths(
  675. repository={'path': 'repo'},
  676. archive='archive',
  677. config={},
  678. local_borg_version=flexmock(),
  679. global_arguments=flexmock(),
  680. local_path=flexmock(),
  681. remote_path=flexmock(),
  682. borgmatic_runtime_directory='/run/user/0/borgmatic',
  683. ) == ('etc/path',)
  684. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory():
  685. flexmock(module.borgmatic.config.paths).should_receive(
  686. 'get_borgmatic_source_directory'
  687. ).and_return('/root.borgmatic')
  688. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  689. (
  690. 'f etc/path',
  691. 'f run/user/0/borgmatic/some/thing',
  692. )
  693. )
  694. assert module.collect_spot_check_archive_paths(
  695. repository={'path': 'repo'},
  696. archive='archive',
  697. config={},
  698. local_borg_version=flexmock(),
  699. global_arguments=flexmock(),
  700. local_path=flexmock(),
  701. remote_path=flexmock(),
  702. borgmatic_runtime_directory='/run/user/0/borgmatic',
  703. ) == ('etc/path',)
  704. def test_collect_spot_check_source_paths_uses_working_directory():
  705. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  706. {'hook1': False, 'hook2': True}
  707. )
  708. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  709. flexmock()
  710. )
  711. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  712. flexmock()
  713. )
  714. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  715. [Pattern('foo'), Pattern('bar')]
  716. )
  717. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  718. dry_run=True,
  719. repository_path='repo',
  720. config=object,
  721. patterns=[Pattern('foo'), Pattern('bar')],
  722. local_borg_version=object,
  723. global_arguments=object,
  724. borgmatic_runtime_directory='/run/borgmatic',
  725. local_path=object,
  726. remote_path=object,
  727. list_files=True,
  728. stream_processes=True,
  729. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  730. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  731. flexmock()
  732. )
  733. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  734. '/working/dir'
  735. )
  736. flexmock(module.borgmatic.execute).should_receive(
  737. 'execute_command_and_capture_output'
  738. ).and_return(
  739. 'warning: stuff\n- foo\n+ bar\n? /nope',
  740. )
  741. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/foo').and_return(True)
  742. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/bar').and_return(True)
  743. assert module.collect_spot_check_source_paths(
  744. repository={'path': 'repo'},
  745. config={'working_directory': '/working/dir'},
  746. local_borg_version=flexmock(),
  747. global_arguments=flexmock(),
  748. local_path=flexmock(),
  749. remote_path=flexmock(),
  750. borgmatic_runtime_directory='/run/borgmatic',
  751. ) == ('foo', 'bar')
  752. def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
  753. flexmock(module.random).should_receive('sample').replace_with(
  754. lambda population, count: population[:count]
  755. )
  756. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  757. None,
  758. )
  759. flexmock(module.os.path).should_receive('exists').and_return(True)
  760. flexmock(module.borgmatic.execute).should_receive(
  761. 'execute_command_and_capture_output'
  762. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  763. 'hash1 /foo\nhash2 /bar'
  764. )
  765. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  766. ['hash1 foo', 'nothash2 bar']
  767. )
  768. assert module.compare_spot_check_hashes(
  769. repository={'path': 'repo'},
  770. archive='archive',
  771. config={
  772. 'checks': [
  773. {
  774. 'name': 'archives',
  775. 'frequency': '2 weeks',
  776. },
  777. {
  778. 'name': 'spot',
  779. 'data_sample_percentage': 50,
  780. },
  781. ]
  782. },
  783. local_borg_version=flexmock(),
  784. global_arguments=flexmock(),
  785. local_path=flexmock(),
  786. remote_path=flexmock(),
  787. source_paths=('/foo', '/bar', '/baz', '/quux'),
  788. ) == ('/bar',)
  789. def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes():
  790. flexmock(module.random).should_receive('sample').replace_with(
  791. lambda population, count: population[:count]
  792. )
  793. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  794. None,
  795. )
  796. flexmock(module.os.path).should_receive('exists').and_return(True)
  797. flexmock(module.borgmatic.execute).should_receive(
  798. 'execute_command_and_capture_output'
  799. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return(
  800. 'hash1 foo\nhash2 bar'
  801. )
  802. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  803. ['hash1 foo', 'nothash2 bar']
  804. )
  805. assert module.compare_spot_check_hashes(
  806. repository={'path': 'repo'},
  807. archive='archive',
  808. config={
  809. 'checks': [
  810. {
  811. 'name': 'archives',
  812. 'frequency': '2 weeks',
  813. },
  814. {
  815. 'name': 'spot',
  816. 'data_sample_percentage': 50,
  817. },
  818. ]
  819. },
  820. local_borg_version=flexmock(),
  821. global_arguments=flexmock(),
  822. local_path=flexmock(),
  823. remote_path=flexmock(),
  824. source_paths=('foo', 'bar', 'baz', 'quux'),
  825. ) == ('bar',)
  826. def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
  827. flexmock(module.random).should_receive('sample').replace_with(
  828. lambda population, count: population[:count]
  829. )
  830. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  831. None,
  832. )
  833. flexmock(module.os.path).should_receive('exists').and_return(True)
  834. flexmock(module.borgmatic.execute).should_receive(
  835. 'execute_command_and_capture_output'
  836. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  837. 'hash1 /foo\nhash2 /bar'
  838. )
  839. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  840. ['nothash1 foo', 'nothash2 bar']
  841. )
  842. assert module.compare_spot_check_hashes(
  843. repository={'path': 'repo'},
  844. archive='archive',
  845. config={
  846. 'checks': [
  847. {
  848. 'name': 'archives',
  849. 'frequency': '2 weeks',
  850. },
  851. {
  852. 'name': 'spot',
  853. 'data_sample_percentage': 1000,
  854. },
  855. ]
  856. },
  857. local_borg_version=flexmock(),
  858. global_arguments=flexmock(),
  859. local_path=flexmock(),
  860. remote_path=flexmock(),
  861. source_paths=('/foo', '/bar'),
  862. ) == ('/foo', '/bar')
  863. def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
  864. flexmock(module.random).should_receive('sample').replace_with(
  865. lambda population, count: population[:count]
  866. )
  867. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  868. None,
  869. )
  870. flexmock(module.os.path).should_receive('exists').and_return(True)
  871. flexmock(module.borgmatic.execute).should_receive(
  872. 'execute_command_and_capture_output'
  873. ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  874. 'hash1 /foo\nhash2 /bar'
  875. )
  876. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  877. ['hash1 foo', 'nothash2 bar']
  878. )
  879. assert module.compare_spot_check_hashes(
  880. repository={'path': 'repo'},
  881. archive='archive',
  882. config={
  883. 'checks': [
  884. {
  885. 'name': 'spot',
  886. 'data_sample_percentage': 50,
  887. 'xxh64sum_command': '/usr/local/bin/xxh64sum',
  888. },
  889. ]
  890. },
  891. local_borg_version=flexmock(),
  892. global_arguments=flexmock(),
  893. local_path=flexmock(),
  894. remote_path=flexmock(),
  895. source_paths=('/foo', '/bar', '/baz', '/quux'),
  896. ) == ('/bar',)
  897. def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_matching():
  898. flexmock(module.random).should_receive('sample').replace_with(
  899. lambda population, count: population[:count]
  900. )
  901. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  902. None,
  903. )
  904. flexmock(module.os.path).should_receive('exists').and_return(True)
  905. flexmock(module.borgmatic.execute).should_receive(
  906. 'execute_command_and_capture_output'
  907. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  908. 'hash1 /foo\nhash2 /bar'
  909. )
  910. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  911. ['hash1 foo']
  912. )
  913. assert module.compare_spot_check_hashes(
  914. repository={'path': 'repo'},
  915. archive='archive',
  916. config={
  917. 'checks': [
  918. {
  919. 'name': 'spot',
  920. 'data_sample_percentage': 50,
  921. },
  922. ]
  923. },
  924. local_borg_version=flexmock(),
  925. global_arguments=flexmock(),
  926. local_path=flexmock(),
  927. remote_path=flexmock(),
  928. source_paths=('/foo', '/bar', '/baz', '/quux'),
  929. ) == ('/bar',)
  930. def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching():
  931. flexmock(module.random).should_receive('sample').replace_with(
  932. lambda population, count: population[:count]
  933. )
  934. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  935. None,
  936. )
  937. flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
  938. flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False)
  939. flexmock(module.borgmatic.execute).should_receive(
  940. 'execute_command_and_capture_output'
  941. ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
  942. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  943. ['hash1 foo', 'hash2 bar']
  944. )
  945. assert module.compare_spot_check_hashes(
  946. repository={'path': 'repo'},
  947. archive='archive',
  948. config={
  949. 'checks': [
  950. {
  951. 'name': 'spot',
  952. 'data_sample_percentage': 50,
  953. },
  954. ]
  955. },
  956. local_borg_version=flexmock(),
  957. global_arguments=flexmock(),
  958. local_path=flexmock(),
  959. remote_path=flexmock(),
  960. source_paths=('/foo', '/bar', '/baz', '/quux'),
  961. ) == ('/bar',)
  962. def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in_chunks():
  963. flexmock(module).SAMPLE_PATHS_SUBSET_COUNT = 2
  964. flexmock(module.random).should_receive('sample').replace_with(
  965. lambda population, count: population[:count]
  966. )
  967. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  968. None,
  969. )
  970. flexmock(module.os.path).should_receive('exists').and_return(True)
  971. flexmock(module.borgmatic.execute).should_receive(
  972. 'execute_command_and_capture_output'
  973. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  974. 'hash1 /foo\nhash2 /bar'
  975. )
  976. flexmock(module.borgmatic.execute).should_receive(
  977. 'execute_command_and_capture_output'
  978. ).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_return(
  979. 'hash3 /baz\nhash4 /quux'
  980. )
  981. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  982. ['hash1 foo', 'hash2 bar']
  983. ).and_return(['hash3 baz', 'nothash4 quux'])
  984. assert module.compare_spot_check_hashes(
  985. repository={'path': 'repo'},
  986. archive='archive',
  987. config={
  988. 'checks': [
  989. {
  990. 'name': 'archives',
  991. 'frequency': '2 weeks',
  992. },
  993. {
  994. 'name': 'spot',
  995. 'data_sample_percentage': 100,
  996. },
  997. ]
  998. },
  999. local_borg_version=flexmock(),
  1000. global_arguments=flexmock(),
  1001. local_path=flexmock(),
  1002. remote_path=flexmock(),
  1003. source_paths=('/foo', '/bar', '/baz', '/quux'),
  1004. ) == ('/quux',)
  1005. def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths():
  1006. flexmock(module.random).should_receive('sample').replace_with(
  1007. lambda population, count: population[:count]
  1008. )
  1009. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  1010. '/working/dir',
  1011. )
  1012. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True)
  1013. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True)
  1014. flexmock(module.borgmatic.execute).should_receive(
  1015. 'execute_command_and_capture_output'
  1016. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return(
  1017. 'hash1 foo\nhash2 bar'
  1018. )
  1019. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  1020. ['hash1 foo', 'nothash2 bar']
  1021. )
  1022. assert module.compare_spot_check_hashes(
  1023. repository={'path': 'repo'},
  1024. archive='archive',
  1025. config={
  1026. 'checks': [
  1027. {
  1028. 'name': 'archives',
  1029. 'frequency': '2 weeks',
  1030. },
  1031. {
  1032. 'name': 'spot',
  1033. 'data_sample_percentage': 50,
  1034. },
  1035. ],
  1036. 'working_directory': '/working/dir',
  1037. },
  1038. local_borg_version=flexmock(),
  1039. global_arguments=flexmock(),
  1040. local_path=flexmock(),
  1041. remote_path=flexmock(),
  1042. source_paths=('foo', 'bar', 'baz', 'quux'),
  1043. ) == ('bar',)
  1044. def test_spot_check_without_spot_configuration_errors():
  1045. with pytest.raises(ValueError):
  1046. module.spot_check(
  1047. repository={'path': 'repo'},
  1048. config={
  1049. 'checks': [
  1050. {
  1051. 'name': 'archives',
  1052. },
  1053. ]
  1054. },
  1055. local_borg_version=flexmock(),
  1056. global_arguments=flexmock(),
  1057. local_path=flexmock(),
  1058. remote_path=flexmock(),
  1059. borgmatic_runtime_directory='/run/borgmatic',
  1060. )
  1061. def test_spot_check_without_any_configuration_errors():
  1062. with pytest.raises(ValueError):
  1063. module.spot_check(
  1064. repository={'path': 'repo'},
  1065. config={},
  1066. local_borg_version=flexmock(),
  1067. global_arguments=flexmock(),
  1068. local_path=flexmock(),
  1069. remote_path=flexmock(),
  1070. borgmatic_runtime_directory='/run/borgmatic',
  1071. )
  1072. def test_spot_check_data_tolerance_percentage_greater_than_data_sample_percentage_errors():
  1073. with pytest.raises(ValueError):
  1074. module.spot_check(
  1075. repository={'path': 'repo'},
  1076. config={
  1077. 'checks': [
  1078. {
  1079. 'name': 'spot',
  1080. 'data_tolerance_percentage': 7,
  1081. 'data_sample_percentage': 5,
  1082. },
  1083. ]
  1084. },
  1085. local_borg_version=flexmock(),
  1086. global_arguments=flexmock(),
  1087. local_path=flexmock(),
  1088. remote_path=flexmock(),
  1089. borgmatic_runtime_directory='/run/borgmatic',
  1090. )
  1091. def test_spot_check_with_count_delta_greater_than_count_tolerance_percentage_errors():
  1092. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1093. ('/foo', '/bar', '/baz', '/quux')
  1094. )
  1095. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1096. 'archive'
  1097. )
  1098. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(
  1099. ('/foo', '/bar')
  1100. ).once()
  1101. with pytest.raises(ValueError):
  1102. module.spot_check(
  1103. repository={'path': 'repo'},
  1104. config={
  1105. 'checks': [
  1106. {
  1107. 'name': 'spot',
  1108. 'count_tolerance_percentage': 1,
  1109. 'data_tolerance_percentage': 4,
  1110. 'data_sample_percentage': 5,
  1111. },
  1112. ]
  1113. },
  1114. local_borg_version=flexmock(),
  1115. global_arguments=flexmock(),
  1116. local_path=flexmock(),
  1117. remote_path=flexmock(),
  1118. borgmatic_runtime_directory='/run/borgmatic',
  1119. )
  1120. def test_spot_check_with_failing_percentage_greater_than_data_tolerance_percentage_errors():
  1121. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1122. ('/foo', '/bar', '/baz', '/quux')
  1123. )
  1124. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1125. 'archive'
  1126. )
  1127. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1128. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1129. ('/bar', '/baz', '/quux')
  1130. ).once()
  1131. with pytest.raises(ValueError):
  1132. module.spot_check(
  1133. repository={'path': 'repo'},
  1134. config={
  1135. 'checks': [
  1136. {
  1137. 'name': 'spot',
  1138. 'count_tolerance_percentage': 55,
  1139. 'data_tolerance_percentage': 4,
  1140. 'data_sample_percentage': 5,
  1141. },
  1142. ]
  1143. },
  1144. local_borg_version=flexmock(),
  1145. global_arguments=flexmock(),
  1146. local_path=flexmock(),
  1147. remote_path=flexmock(),
  1148. borgmatic_runtime_directory='/run/borgmatic',
  1149. )
  1150. def test_spot_check_with_high_enough_tolerances_does_not_raise():
  1151. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1152. ('/foo', '/bar', '/baz', '/quux')
  1153. )
  1154. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1155. 'archive'
  1156. )
  1157. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1158. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1159. ('/bar', '/baz', '/quux')
  1160. ).once()
  1161. module.spot_check(
  1162. repository={'path': 'repo'},
  1163. config={
  1164. 'checks': [
  1165. {
  1166. 'name': 'spot',
  1167. 'count_tolerance_percentage': 55,
  1168. 'data_tolerance_percentage': 80,
  1169. 'data_sample_percentage': 80,
  1170. },
  1171. ]
  1172. },
  1173. local_borg_version=flexmock(),
  1174. global_arguments=flexmock(),
  1175. local_path=flexmock(),
  1176. remote_path=flexmock(),
  1177. borgmatic_runtime_directory='/run/borgmatic',
  1178. )
  1179. def test_spot_check_without_any_source_paths_errors():
  1180. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(())
  1181. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1182. 'archive'
  1183. )
  1184. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1185. flexmock(module).should_receive('compare_spot_check_hashes').never()
  1186. with pytest.raises(ValueError):
  1187. module.spot_check(
  1188. repository={'path': 'repo'},
  1189. config={
  1190. 'checks': [
  1191. {
  1192. 'name': 'spot',
  1193. 'count_tolerance_percentage': 10,
  1194. 'data_tolerance_percentage': 40,
  1195. 'data_sample_percentage': 50,
  1196. },
  1197. ]
  1198. },
  1199. local_borg_version=flexmock(),
  1200. global_arguments=flexmock(),
  1201. local_path=flexmock(),
  1202. remote_path=flexmock(),
  1203. borgmatic_runtime_directory='/run/borgmatic',
  1204. )
  1205. def test_run_check_checks_archives_for_configured_repository():
  1206. flexmock(module.logger).answer = lambda message: None
  1207. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1208. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1209. flexmock(module).should_receive('upgrade_check_times')
  1210. flexmock(module).should_receive('parse_checks')
  1211. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1212. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1213. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1214. {'repository', 'archives'}
  1215. )
  1216. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1217. flexmock(module).should_receive('make_check_time_path')
  1218. flexmock(module).should_receive('write_check_time')
  1219. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1220. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1221. check_arguments = flexmock(
  1222. repository=None,
  1223. progress=flexmock(),
  1224. repair=flexmock(),
  1225. only_checks=flexmock(),
  1226. force=flexmock(),
  1227. )
  1228. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1229. module.run_check(
  1230. config_filename='test.yaml',
  1231. repository={'path': 'repo'},
  1232. config={'repositories': ['repo']},
  1233. hook_context={},
  1234. local_borg_version=None,
  1235. check_arguments=check_arguments,
  1236. global_arguments=global_arguments,
  1237. local_path=None,
  1238. remote_path=None,
  1239. )
  1240. def test_run_check_runs_configured_extract_check():
  1241. flexmock(module.logger).answer = lambda message: None
  1242. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1243. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1244. flexmock(module).should_receive('upgrade_check_times')
  1245. flexmock(module).should_receive('parse_checks')
  1246. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1247. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1248. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'extract'})
  1249. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1250. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').once()
  1251. flexmock(module).should_receive('make_check_time_path')
  1252. flexmock(module).should_receive('write_check_time')
  1253. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1254. check_arguments = flexmock(
  1255. repository=None,
  1256. progress=flexmock(),
  1257. repair=flexmock(),
  1258. only_checks=flexmock(),
  1259. force=flexmock(),
  1260. )
  1261. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1262. module.run_check(
  1263. config_filename='test.yaml',
  1264. repository={'path': 'repo'},
  1265. config={'repositories': ['repo']},
  1266. hook_context={},
  1267. local_borg_version=None,
  1268. check_arguments=check_arguments,
  1269. global_arguments=global_arguments,
  1270. local_path=None,
  1271. remote_path=None,
  1272. )
  1273. def test_run_check_runs_configured_spot_check():
  1274. flexmock(module.logger).answer = lambda message: None
  1275. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1276. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1277. flexmock(module).should_receive('upgrade_check_times')
  1278. flexmock(module).should_receive('parse_checks')
  1279. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1280. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1281. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'spot'})
  1282. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1283. flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
  1284. flexmock()
  1285. )
  1286. flexmock(module.borgmatic.actions.check).should_receive('spot_check').once()
  1287. flexmock(module).should_receive('make_check_time_path')
  1288. flexmock(module).should_receive('write_check_time')
  1289. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1290. check_arguments = flexmock(
  1291. repository=None,
  1292. progress=flexmock(),
  1293. repair=flexmock(),
  1294. only_checks=flexmock(),
  1295. force=flexmock(),
  1296. )
  1297. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1298. module.run_check(
  1299. config_filename='test.yaml',
  1300. repository={'path': 'repo'},
  1301. config={'repositories': ['repo']},
  1302. hook_context={},
  1303. local_borg_version=None,
  1304. check_arguments=check_arguments,
  1305. global_arguments=global_arguments,
  1306. local_path=None,
  1307. remote_path=None,
  1308. )
  1309. def test_run_check_without_checks_runs_nothing_except_hooks():
  1310. flexmock(module.logger).answer = lambda message: None
  1311. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1312. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1313. flexmock(module).should_receive('upgrade_check_times')
  1314. flexmock(module).should_receive('parse_checks')
  1315. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1316. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1317. flexmock(module).should_receive('filter_checks_on_frequency').and_return({})
  1318. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1319. flexmock(module).should_receive('make_check_time_path')
  1320. flexmock(module).should_receive('write_check_time').never()
  1321. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1322. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1323. check_arguments = flexmock(
  1324. repository=None,
  1325. progress=flexmock(),
  1326. repair=flexmock(),
  1327. only_checks=flexmock(),
  1328. force=flexmock(),
  1329. )
  1330. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1331. module.run_check(
  1332. config_filename='test.yaml',
  1333. repository={'path': 'repo'},
  1334. config={'repositories': ['repo']},
  1335. hook_context={},
  1336. local_borg_version=None,
  1337. check_arguments=check_arguments,
  1338. global_arguments=global_arguments,
  1339. local_path=None,
  1340. remote_path=None,
  1341. )
  1342. def test_run_check_checks_archives_in_selected_repository():
  1343. flexmock(module.logger).answer = lambda message: None
  1344. flexmock(module.borgmatic.config.validate).should_receive(
  1345. 'repositories_match'
  1346. ).once().and_return(True)
  1347. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1348. flexmock(module).should_receive('upgrade_check_times')
  1349. flexmock(module).should_receive('parse_checks')
  1350. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1351. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1352. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1353. {'repository', 'archives'}
  1354. )
  1355. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1356. flexmock(module).should_receive('make_check_time_path')
  1357. flexmock(module).should_receive('write_check_time')
  1358. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1359. check_arguments = flexmock(
  1360. repository=flexmock(),
  1361. progress=flexmock(),
  1362. repair=flexmock(),
  1363. only_checks=flexmock(),
  1364. force=flexmock(),
  1365. )
  1366. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1367. module.run_check(
  1368. config_filename='test.yaml',
  1369. repository={'path': 'repo'},
  1370. config={'repositories': ['repo']},
  1371. hook_context={},
  1372. local_borg_version=None,
  1373. check_arguments=check_arguments,
  1374. global_arguments=global_arguments,
  1375. local_path=None,
  1376. remote_path=None,
  1377. )
  1378. def test_run_check_bails_if_repository_does_not_match():
  1379. flexmock(module.logger).answer = lambda message: None
  1380. flexmock(module.borgmatic.config.validate).should_receive(
  1381. 'repositories_match'
  1382. ).once().and_return(False)
  1383. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1384. check_arguments = flexmock(
  1385. repository=flexmock(),
  1386. progress=flexmock(),
  1387. repair=flexmock(),
  1388. only_checks=flexmock(),
  1389. force=flexmock(),
  1390. )
  1391. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1392. module.run_check(
  1393. config_filename='test.yaml',
  1394. repository={'path': 'repo'},
  1395. config={'repositories': ['repo']},
  1396. hook_context={},
  1397. local_borg_version=None,
  1398. check_arguments=check_arguments,
  1399. global_arguments=global_arguments,
  1400. local_path=None,
  1401. remote_path=None,
  1402. )