test_check.py 60 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614
  1. import pytest
  2. from flexmock import flexmock
  3. from borgmatic.actions import check as module
  4. from borgmatic.borg.pattern import Pattern
  5. def test_parse_checks_returns_them_as_tuple():
  6. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'bar'}]})
  7. assert checks == ('foo', 'bar')
  8. def test_parse_checks_with_missing_value_returns_defaults():
  9. checks = module.parse_checks({})
  10. assert checks == ('repository', 'archives')
  11. def test_parse_checks_with_empty_list_returns_defaults():
  12. checks = module.parse_checks({'checks': []})
  13. assert checks == ('repository', 'archives')
  14. def test_parse_checks_with_none_value_returns_defaults():
  15. checks = module.parse_checks({'checks': None})
  16. assert checks == ('repository', 'archives')
  17. def test_parse_checks_with_disabled_returns_no_checks():
  18. checks = module.parse_checks({'checks': [{'name': 'foo'}, {'name': 'disabled'}]})
  19. assert checks == ()
  20. def test_parse_checks_prefers_override_checks_to_configured_checks():
  21. checks = module.parse_checks(
  22. {'checks': [{'name': 'archives'}]}, only_checks=['repository', 'extract']
  23. )
  24. assert checks == ('repository', 'extract')
  25. @pytest.mark.parametrize(
  26. 'frequency,expected_result',
  27. (
  28. (None, None),
  29. ('always', None),
  30. ('1 hour', module.datetime.timedelta(hours=1)),
  31. ('2 hours', module.datetime.timedelta(hours=2)),
  32. ('1 day', module.datetime.timedelta(days=1)),
  33. ('2 days', module.datetime.timedelta(days=2)),
  34. ('1 week', module.datetime.timedelta(weeks=1)),
  35. ('2 weeks', module.datetime.timedelta(weeks=2)),
  36. ('1 month', module.datetime.timedelta(days=30)),
  37. ('2 months', module.datetime.timedelta(days=60)),
  38. ('1 year', module.datetime.timedelta(days=365)),
  39. ('2 years', module.datetime.timedelta(days=365 * 2)),
  40. ),
  41. )
  42. def test_parse_frequency_parses_into_timedeltas(frequency, expected_result):
  43. assert module.parse_frequency(frequency) == expected_result
  44. @pytest.mark.parametrize(
  45. 'frequency',
  46. (
  47. 'sometime',
  48. 'x days',
  49. '3 decades',
  50. ),
  51. )
  52. def test_parse_frequency_raises_on_parse_error(frequency):
  53. with pytest.raises(ValueError):
  54. module.parse_frequency(frequency)
  55. def test_filter_checks_on_frequency_without_config_uses_default_checks():
  56. flexmock(module).should_receive('parse_frequency').and_return(
  57. module.datetime.timedelta(weeks=4)
  58. )
  59. flexmock(module).should_receive('make_check_time_path')
  60. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  61. assert module.filter_checks_on_frequency(
  62. config={},
  63. borg_repository_id='repo',
  64. checks=('repository', 'archives'),
  65. force=False,
  66. archives_check_id='1234',
  67. ) == ('repository', 'archives')
  68. def test_filter_checks_on_frequency_retains_unconfigured_check():
  69. assert module.filter_checks_on_frequency(
  70. config={},
  71. borg_repository_id='repo',
  72. checks=('data',),
  73. force=False,
  74. ) == ('data',)
  75. def test_filter_checks_on_frequency_retains_check_without_frequency():
  76. flexmock(module).should_receive('parse_frequency').and_return(None)
  77. assert module.filter_checks_on_frequency(
  78. config={'checks': [{'name': 'archives'}]},
  79. borg_repository_id='repo',
  80. checks=('archives',),
  81. force=False,
  82. archives_check_id='1234',
  83. ) == ('archives',)
  84. def test_filter_checks_on_frequency_retains_check_with_empty_only_run_on():
  85. flexmock(module).should_receive('parse_frequency').and_return(None)
  86. assert module.filter_checks_on_frequency(
  87. config={'checks': [{'name': 'archives', 'only_run_on': []}]},
  88. borg_repository_id='repo',
  89. checks=('archives',),
  90. force=False,
  91. archives_check_id='1234',
  92. datetime_now=flexmock(weekday=lambda: 0),
  93. ) == ('archives',)
  94. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today():
  95. flexmock(module).should_receive('parse_frequency').and_return(None)
  96. assert module.filter_checks_on_frequency(
  97. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[0]]}]},
  98. borg_repository_id='repo',
  99. checks=('archives',),
  100. force=False,
  101. archives_check_id='1234',
  102. datetime_now=flexmock(weekday=lambda: 0),
  103. ) == ('archives',)
  104. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekday_value():
  105. flexmock(module).should_receive('parse_frequency').and_return(None)
  106. assert module.filter_checks_on_frequency(
  107. config={'checks': [{'name': 'archives', 'only_run_on': ['weekday']}]},
  108. borg_repository_id='repo',
  109. checks=('archives',),
  110. force=False,
  111. archives_check_id='1234',
  112. datetime_now=flexmock(weekday=lambda: 0),
  113. ) == ('archives',)
  114. def test_filter_checks_on_frequency_retains_check_with_only_run_on_matching_today_via_weekend_value():
  115. flexmock(module).should_receive('parse_frequency').and_return(None)
  116. assert module.filter_checks_on_frequency(
  117. config={'checks': [{'name': 'archives', 'only_run_on': ['weekend']}]},
  118. borg_repository_id='repo',
  119. checks=('archives',),
  120. force=False,
  121. archives_check_id='1234',
  122. datetime_now=flexmock(weekday=lambda: 6),
  123. ) == ('archives',)
  124. def test_filter_checks_on_frequency_skips_check_with_only_run_on_not_matching_today():
  125. flexmock(module).should_receive('parse_frequency').and_return(None)
  126. assert (
  127. module.filter_checks_on_frequency(
  128. config={'checks': [{'name': 'archives', 'only_run_on': [module.calendar.day_name[5]]}]},
  129. borg_repository_id='repo',
  130. checks=('archives',),
  131. force=False,
  132. archives_check_id='1234',
  133. datetime_now=flexmock(weekday=lambda: 0),
  134. )
  135. == ()
  136. )
  137. def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency():
  138. flexmock(module).should_receive('parse_frequency').and_return(
  139. module.datetime.timedelta(hours=1)
  140. )
  141. flexmock(module).should_receive('make_check_time_path')
  142. flexmock(module).should_receive('probe_for_check_time').and_return(
  143. module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1)
  144. )
  145. assert module.filter_checks_on_frequency(
  146. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  147. borg_repository_id='repo',
  148. checks=('archives',),
  149. force=False,
  150. archives_check_id='1234',
  151. ) == ('archives',)
  152. def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file():
  153. flexmock(module).should_receive('parse_frequency').and_return(
  154. module.datetime.timedelta(hours=1)
  155. )
  156. flexmock(module).should_receive('make_check_time_path')
  157. flexmock(module).should_receive('probe_for_check_time').and_return(None)
  158. assert module.filter_checks_on_frequency(
  159. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  160. borg_repository_id='repo',
  161. checks=('archives',),
  162. force=False,
  163. archives_check_id='1234',
  164. ) == ('archives',)
  165. def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency():
  166. flexmock(module).should_receive('parse_frequency').and_return(
  167. module.datetime.timedelta(hours=1)
  168. )
  169. flexmock(module).should_receive('make_check_time_path')
  170. flexmock(module).should_receive('probe_for_check_time').and_return(
  171. module.datetime.datetime.now()
  172. )
  173. assert (
  174. module.filter_checks_on_frequency(
  175. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  176. borg_repository_id='repo',
  177. checks=('archives',),
  178. force=False,
  179. archives_check_id='1234',
  180. )
  181. == ()
  182. )
  183. def test_filter_checks_on_frequency_retains_check_with_unelapsed_frequency_and_force():
  184. assert module.filter_checks_on_frequency(
  185. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  186. borg_repository_id='repo',
  187. checks=('archives',),
  188. force=True,
  189. archives_check_id='1234',
  190. ) == ('archives',)
  191. def test_filter_checks_on_frequency_passes_through_empty_checks():
  192. assert (
  193. module.filter_checks_on_frequency(
  194. config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
  195. borg_repository_id='repo',
  196. checks=(),
  197. force=False,
  198. archives_check_id='1234',
  199. )
  200. == ()
  201. )
  202. def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise():
  203. assert module.make_archives_check_id(('--match-archives', 'sh:foo-*'))
  204. def test_make_archives_check_id_with_empty_flags_returns_none():
  205. assert module.make_archives_check_id(()) is None
  206. def test_make_check_time_path_with_borgmatic_source_directory_includes_it():
  207. flexmock(module.borgmatic.config.paths).should_receive(
  208. 'get_borgmatic_state_directory'
  209. ).and_return(
  210. '/home/user/.local/state/borgmatic',
  211. )
  212. assert (
  213. module.make_check_time_path({}, '1234', 'archives', '5678')
  214. == '/home/user/.local/state/borgmatic/checks/1234/archives/5678'
  215. )
  216. def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all():
  217. flexmock(module.borgmatic.config.paths).should_receive(
  218. 'get_borgmatic_state_directory'
  219. ).and_return(
  220. '/home/user/.local/state/borgmatic',
  221. )
  222. assert (
  223. module.make_check_time_path(
  224. {},
  225. '1234',
  226. 'archives',
  227. )
  228. == '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  229. )
  230. def test_make_check_time_path_with_repositories_check_ignores_archives_check_id():
  231. flexmock(module.borgmatic.config.paths).should_receive(
  232. 'get_borgmatic_state_directory'
  233. ).and_return(
  234. '/home/user/.local/state/borgmatic',
  235. )
  236. assert (
  237. module.make_check_time_path({}, '1234', 'repository', '5678')
  238. == '/home/user/.local/state/borgmatic/checks/1234/repository'
  239. )
  240. def test_read_check_time_does_not_raise():
  241. flexmock(module.os).should_receive('stat').and_return(flexmock(st_mtime=123))
  242. assert module.read_check_time('/path')
  243. def test_read_check_time_on_missing_file_does_not_raise():
  244. flexmock(module.os).should_receive('stat').and_raise(FileNotFoundError)
  245. assert module.read_check_time('/path') is None
  246. def test_probe_for_check_time_uses_maximum_of_multiple_check_times():
  247. flexmock(module).should_receive('make_check_time_path').and_return(
  248. '~/.borgmatic/checks/1234/archives/5678'
  249. ).and_return('~/.borgmatic/checks/1234/archives/all')
  250. flexmock(module).should_receive('read_check_time').and_return(1).and_return(2)
  251. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  252. def test_probe_for_check_time_deduplicates_identical_check_time_paths():
  253. flexmock(module).should_receive('make_check_time_path').and_return(
  254. '~/.borgmatic/checks/1234/archives/5678'
  255. ).and_return('~/.borgmatic/checks/1234/archives/5678')
  256. flexmock(module).should_receive('read_check_time').and_return(1).once()
  257. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  258. def test_probe_for_check_time_skips_none_check_time():
  259. flexmock(module).should_receive('make_check_time_path').and_return(
  260. '~/.borgmatic/checks/1234/archives/5678'
  261. ).and_return('~/.borgmatic/checks/1234/archives/all')
  262. flexmock(module).should_receive('read_check_time').and_return(None).and_return(2)
  263. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2
  264. def test_probe_for_check_time_uses_single_check_time():
  265. flexmock(module).should_receive('make_check_time_path').and_return(
  266. '~/.borgmatic/checks/1234/archives/5678'
  267. ).and_return('~/.borgmatic/checks/1234/archives/all')
  268. flexmock(module).should_receive('read_check_time').and_return(1).and_return(None)
  269. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1
  270. def test_probe_for_check_time_returns_none_when_no_check_time_found():
  271. flexmock(module).should_receive('make_check_time_path').and_return(
  272. '~/.borgmatic/checks/1234/archives/5678'
  273. ).and_return('~/.borgmatic/checks/1234/archives/all')
  274. flexmock(module).should_receive('read_check_time').and_return(None).and_return(None)
  275. assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None
  276. def test_upgrade_check_times_moves_checks_from_borgmatic_source_directory_to_state_directory():
  277. flexmock(module.borgmatic.config.paths).should_receive(
  278. 'get_borgmatic_source_directory'
  279. ).and_return('/home/user/.borgmatic')
  280. flexmock(module.borgmatic.config.paths).should_receive(
  281. 'get_borgmatic_state_directory'
  282. ).and_return('/home/user/.local/state/borgmatic')
  283. flexmock(module.os.path).should_receive('exists').with_args(
  284. '/home/user/.borgmatic/checks'
  285. ).and_return(True)
  286. flexmock(module.os.path).should_receive('exists').with_args(
  287. '/home/user/.local/state/borgmatic/checks'
  288. ).and_return(False)
  289. flexmock(module.os).should_receive('makedirs')
  290. flexmock(module.shutil).should_receive('move').with_args(
  291. '/home/user/.borgmatic/checks', '/home/user/.local/state/borgmatic/checks'
  292. ).once()
  293. flexmock(module).should_receive('make_check_time_path').and_return(
  294. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  295. )
  296. flexmock(module.os.path).should_receive('isfile').and_return(False)
  297. flexmock(module.os).should_receive('mkdir').never()
  298. module.upgrade_check_times(flexmock(), flexmock())
  299. def test_upgrade_check_times_with_checks_already_in_borgmatic_state_directory_does_not_move_anything():
  300. flexmock(module.borgmatic.config.paths).should_receive(
  301. 'get_borgmatic_source_directory'
  302. ).and_return('/home/user/.borgmatic')
  303. flexmock(module.borgmatic.config.paths).should_receive(
  304. 'get_borgmatic_state_directory'
  305. ).and_return('/home/user/.local/state/borgmatic')
  306. flexmock(module.os.path).should_receive('exists').with_args(
  307. '/home/user/.borgmatic/checks'
  308. ).and_return(True)
  309. flexmock(module.os.path).should_receive('exists').with_args(
  310. '/home/user/.local/state/borgmatic/checks'
  311. ).and_return(True)
  312. flexmock(module.os).should_receive('makedirs').never()
  313. flexmock(module.shutil).should_receive('move').never()
  314. flexmock(module).should_receive('make_check_time_path').and_return(
  315. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  316. )
  317. flexmock(module.os.path).should_receive('isfile').and_return(False)
  318. flexmock(module.shutil).should_receive('move').never()
  319. flexmock(module.os).should_receive('mkdir').never()
  320. module.upgrade_check_times(flexmock(), flexmock())
  321. def test_upgrade_check_times_renames_old_check_paths_to_all():
  322. flexmock(module.borgmatic.config.paths).should_receive(
  323. 'get_borgmatic_source_directory'
  324. ).and_return('/home/user/.borgmatic')
  325. flexmock(module.borgmatic.config.paths).should_receive(
  326. 'get_borgmatic_state_directory'
  327. ).and_return('/home/user/.local/state/borgmatic')
  328. flexmock(module.os.path).should_receive('exists').and_return(False)
  329. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  330. flexmock(module).should_receive('make_check_time_path').with_args(
  331. object, object, 'archives', 'all'
  332. ).and_return(f'{base_path}/archives/all')
  333. flexmock(module).should_receive('make_check_time_path').with_args(
  334. object, object, 'data', 'all'
  335. ).and_return(f'{base_path}/data/all')
  336. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  337. True
  338. )
  339. flexmock(module.os.path).should_receive('isfile').with_args(
  340. f'{base_path}/archives.temp'
  341. ).and_return(False)
  342. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  343. False
  344. )
  345. flexmock(module.os.path).should_receive('isfile').with_args(
  346. f'{base_path}/data.temp'
  347. ).and_return(False)
  348. flexmock(module.shutil).should_receive('move').with_args(
  349. f'{base_path}/archives', f'{base_path}/archives.temp'
  350. ).once()
  351. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  352. flexmock(module.shutil).should_receive('move').with_args(
  353. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  354. ).once()
  355. module.upgrade_check_times(flexmock(), flexmock())
  356. def test_upgrade_check_times_renames_data_check_paths_when_archives_paths_are_already_upgraded():
  357. flexmock(module.borgmatic.config.paths).should_receive(
  358. 'get_borgmatic_source_directory'
  359. ).and_return('/home/user/.borgmatic')
  360. flexmock(module.borgmatic.config.paths).should_receive(
  361. 'get_borgmatic_state_directory'
  362. ).and_return('/home/user/.local/state/borgmatic')
  363. flexmock(module.os.path).should_receive('exists').and_return(False)
  364. base_path = '/home/user/.local/state/borgmatic/checks/1234'
  365. flexmock(module).should_receive('make_check_time_path').with_args(
  366. object, object, 'archives', 'all'
  367. ).and_return(f'{base_path}/archives/all')
  368. flexmock(module).should_receive('make_check_time_path').with_args(
  369. object, object, 'data', 'all'
  370. ).and_return(f'{base_path}/data/all')
  371. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  372. False
  373. )
  374. flexmock(module.os.path).should_receive('isfile').with_args(
  375. f'{base_path}/archives.temp'
  376. ).and_return(False)
  377. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  378. True
  379. )
  380. flexmock(module.shutil).should_receive('move').with_args(
  381. f'{base_path}/data', f'{base_path}/data.temp'
  382. ).once()
  383. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/data').once()
  384. flexmock(module.shutil).should_receive('move').with_args(
  385. f'{base_path}/data.temp', f'{base_path}/data/all'
  386. ).once()
  387. module.upgrade_check_times(flexmock(), flexmock())
  388. def test_upgrade_check_times_skips_already_upgraded_check_paths():
  389. flexmock(module.borgmatic.config.paths).should_receive(
  390. 'get_borgmatic_source_directory'
  391. ).and_return('/home/user/.borgmatic')
  392. flexmock(module.borgmatic.config.paths).should_receive(
  393. 'get_borgmatic_state_directory'
  394. ).and_return('/home/user/.local/state/borgmatic')
  395. flexmock(module.os.path).should_receive('exists').and_return(False)
  396. flexmock(module).should_receive('make_check_time_path').and_return(
  397. '/home/user/.local/state/borgmatic/checks/1234/archives/all'
  398. )
  399. flexmock(module.os.path).should_receive('isfile').and_return(False)
  400. flexmock(module.shutil).should_receive('move').never()
  401. flexmock(module.os).should_receive('mkdir').never()
  402. module.upgrade_check_times(flexmock(), flexmock())
  403. def test_upgrade_check_times_renames_stale_temporary_check_path():
  404. flexmock(module.borgmatic.config.paths).should_receive(
  405. 'get_borgmatic_source_directory'
  406. ).and_return('/home/user/.borgmatic')
  407. flexmock(module.borgmatic.config.paths).should_receive(
  408. 'get_borgmatic_state_directory'
  409. ).and_return('/home/user/.local/state/borgmatic')
  410. flexmock(module.os.path).should_receive('exists').and_return(False)
  411. base_path = '/home/borgmatic/.local/state/checks/1234'
  412. flexmock(module).should_receive('make_check_time_path').with_args(
  413. object, object, 'archives', 'all'
  414. ).and_return(f'{base_path}/archives/all')
  415. flexmock(module).should_receive('make_check_time_path').with_args(
  416. object, object, 'data', 'all'
  417. ).and_return(f'{base_path}/data/all')
  418. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return(
  419. False
  420. )
  421. flexmock(module.os.path).should_receive('isfile').with_args(
  422. f'{base_path}/archives.temp'
  423. ).and_return(True)
  424. flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return(
  425. False
  426. )
  427. flexmock(module.os.path).should_receive('isfile').with_args(
  428. f'{base_path}/data.temp'
  429. ).and_return(False)
  430. flexmock(module.shutil).should_receive('move').with_args(
  431. f'{base_path}/archives', f'{base_path}/archives.temp'
  432. ).and_raise(FileNotFoundError)
  433. flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once()
  434. flexmock(module.shutil).should_receive('move').with_args(
  435. f'{base_path}/archives.temp', f'{base_path}/archives/all'
  436. ).once()
  437. module.upgrade_check_times(flexmock(), flexmock())
  438. def test_collect_spot_check_source_paths_parses_borg_output():
  439. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  440. {'hook1': False, 'hook2': True}
  441. )
  442. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  443. flexmock()
  444. )
  445. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  446. flexmock()
  447. )
  448. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  449. [Pattern('foo'), Pattern('bar')]
  450. )
  451. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  452. dry_run=True,
  453. repository_path='repo',
  454. config=object,
  455. patterns=[Pattern('foo'), Pattern('bar')],
  456. local_borg_version=object,
  457. global_arguments=object,
  458. borgmatic_runtime_directory='/run/borgmatic',
  459. local_path=object,
  460. remote_path=object,
  461. list_files=True,
  462. stream_processes=True,
  463. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  464. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  465. flexmock()
  466. )
  467. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  468. flexmock(module.borgmatic.execute).should_receive(
  469. 'execute_command_and_capture_output'
  470. ).and_return(
  471. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  472. )
  473. flexmock(module.os.path).should_receive('isfile').and_return(True)
  474. assert module.collect_spot_check_source_paths(
  475. repository={'path': 'repo'},
  476. config={'working_directory': '/'},
  477. local_borg_version=flexmock(),
  478. global_arguments=flexmock(),
  479. local_path=flexmock(),
  480. remote_path=flexmock(),
  481. borgmatic_runtime_directory='/run/borgmatic',
  482. ) == ('/etc/path', '/etc/other')
  483. def test_collect_spot_check_source_paths_passes_through_stream_processes_false():
  484. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  485. {'hook1': False, 'hook2': False}
  486. )
  487. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  488. flexmock()
  489. )
  490. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  491. flexmock()
  492. )
  493. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  494. [Pattern('foo'), Pattern('bar')]
  495. )
  496. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  497. dry_run=True,
  498. repository_path='repo',
  499. config=object,
  500. patterns=[Pattern('foo'), Pattern('bar')],
  501. local_borg_version=object,
  502. global_arguments=object,
  503. borgmatic_runtime_directory='/run/borgmatic',
  504. local_path=object,
  505. remote_path=object,
  506. list_files=True,
  507. stream_processes=False,
  508. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  509. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  510. flexmock()
  511. )
  512. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  513. flexmock(module.borgmatic.execute).should_receive(
  514. 'execute_command_and_capture_output'
  515. ).and_return(
  516. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  517. )
  518. flexmock(module.os.path).should_receive('isfile').and_return(True)
  519. assert module.collect_spot_check_source_paths(
  520. repository={'path': 'repo'},
  521. config={'working_directory': '/'},
  522. local_borg_version=flexmock(),
  523. global_arguments=flexmock(),
  524. local_path=flexmock(),
  525. remote_path=flexmock(),
  526. borgmatic_runtime_directory='/run/borgmatic',
  527. ) == ('/etc/path', '/etc/other')
  528. def test_collect_spot_check_source_paths_without_working_directory_parses_borg_output():
  529. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  530. {'hook1': False, 'hook2': True}
  531. )
  532. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  533. flexmock()
  534. )
  535. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  536. flexmock()
  537. )
  538. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  539. [Pattern('foo'), Pattern('bar')]
  540. )
  541. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  542. dry_run=True,
  543. repository_path='repo',
  544. config=object,
  545. patterns=[Pattern('foo'), Pattern('bar')],
  546. local_borg_version=object,
  547. global_arguments=object,
  548. borgmatic_runtime_directory='/run/borgmatic',
  549. local_path=object,
  550. remote_path=object,
  551. list_files=True,
  552. stream_processes=True,
  553. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  554. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  555. flexmock()
  556. )
  557. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  558. flexmock(module.borgmatic.execute).should_receive(
  559. 'execute_command_and_capture_output'
  560. ).and_return(
  561. 'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
  562. )
  563. flexmock(module.os.path).should_receive('isfile').and_return(True)
  564. assert module.collect_spot_check_source_paths(
  565. repository={'path': 'repo'},
  566. config={},
  567. local_borg_version=flexmock(),
  568. global_arguments=flexmock(),
  569. local_path=flexmock(),
  570. remote_path=flexmock(),
  571. borgmatic_runtime_directory='/run/borgmatic',
  572. ) == ('/etc/path', '/etc/other')
  573. def test_collect_spot_check_source_paths_skips_directories():
  574. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  575. {'hook1': False, 'hook2': True}
  576. )
  577. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  578. flexmock()
  579. )
  580. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  581. flexmock()
  582. )
  583. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  584. [Pattern('foo'), Pattern('bar')]
  585. )
  586. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  587. dry_run=True,
  588. repository_path='repo',
  589. config=object,
  590. patterns=[Pattern('foo'), Pattern('bar')],
  591. local_borg_version=object,
  592. global_arguments=object,
  593. borgmatic_runtime_directory='/run/borgmatic',
  594. local_path=object,
  595. remote_path=object,
  596. list_files=True,
  597. stream_processes=True,
  598. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  599. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  600. flexmock()
  601. )
  602. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  603. flexmock(module.borgmatic.execute).should_receive(
  604. 'execute_command_and_capture_output'
  605. ).and_return(
  606. 'warning: stuff\n- /etc/path\n+ /etc/dir\n? /nope',
  607. )
  608. flexmock(module.os.path).should_receive('isfile').with_args('/etc/path').and_return(False)
  609. flexmock(module.os.path).should_receive('isfile').with_args('/etc/dir').and_return(False)
  610. assert (
  611. module.collect_spot_check_source_paths(
  612. repository={'path': 'repo'},
  613. config={'working_directory': '/'},
  614. local_borg_version=flexmock(),
  615. global_arguments=flexmock(),
  616. local_path=flexmock(),
  617. remote_path=flexmock(),
  618. borgmatic_runtime_directory='/run/borgmatic',
  619. )
  620. == ()
  621. )
  622. def test_collect_spot_check_archive_paths_excludes_directories_and_pipes():
  623. flexmock(module.borgmatic.config.paths).should_receive(
  624. 'get_borgmatic_source_directory'
  625. ).and_return('/home/user/.borgmatic')
  626. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  627. (
  628. 'f etc/path',
  629. 'p var/pipe',
  630. 'f etc/other',
  631. 'd etc/dir',
  632. )
  633. )
  634. assert module.collect_spot_check_archive_paths(
  635. repository={'path': 'repo'},
  636. archive='archive',
  637. config={},
  638. local_borg_version=flexmock(),
  639. global_arguments=flexmock(),
  640. local_path=flexmock(),
  641. remote_path=flexmock(),
  642. borgmatic_runtime_directory='/run/user/1001/borgmatic',
  643. ) == ('etc/path', 'etc/other')
  644. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory_as_stored_with_prefix_truncation():
  645. flexmock(module.borgmatic.config.paths).should_receive(
  646. 'get_borgmatic_source_directory'
  647. ).and_return('/root/.borgmatic')
  648. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  649. (
  650. 'f etc/path',
  651. 'f borgmatic/some/thing',
  652. )
  653. )
  654. assert module.collect_spot_check_archive_paths(
  655. repository={'path': 'repo'},
  656. archive='archive',
  657. config={},
  658. local_borg_version=flexmock(),
  659. global_arguments=flexmock(),
  660. local_path=flexmock(),
  661. remote_path=flexmock(),
  662. borgmatic_runtime_directory='/run/user/0/borgmatic',
  663. ) == ('etc/path',)
  664. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_source_directory():
  665. flexmock(module.borgmatic.config.paths).should_receive(
  666. 'get_borgmatic_source_directory'
  667. ).and_return('/root/.borgmatic')
  668. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  669. (
  670. 'f etc/path',
  671. 'f root/.borgmatic/some/thing',
  672. )
  673. )
  674. assert module.collect_spot_check_archive_paths(
  675. repository={'path': 'repo'},
  676. archive='archive',
  677. config={},
  678. local_borg_version=flexmock(),
  679. global_arguments=flexmock(),
  680. local_path=flexmock(),
  681. remote_path=flexmock(),
  682. borgmatic_runtime_directory='/run/user/0/borgmatic',
  683. ) == ('etc/path',)
  684. def test_collect_spot_check_archive_paths_excludes_file_in_borgmatic_runtime_directory():
  685. flexmock(module.borgmatic.config.paths).should_receive(
  686. 'get_borgmatic_source_directory'
  687. ).and_return('/root.borgmatic')
  688. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  689. (
  690. 'f etc/path',
  691. 'f run/user/0/borgmatic/some/thing',
  692. )
  693. )
  694. assert module.collect_spot_check_archive_paths(
  695. repository={'path': 'repo'},
  696. archive='archive',
  697. config={},
  698. local_borg_version=flexmock(),
  699. global_arguments=flexmock(),
  700. local_path=flexmock(),
  701. remote_path=flexmock(),
  702. borgmatic_runtime_directory='/run/user/0/borgmatic',
  703. ) == ('etc/path',)
  704. def test_collect_spot_check_source_paths_uses_working_directory():
  705. flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
  706. {'hook1': False, 'hook2': True}
  707. )
  708. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  709. flexmock()
  710. )
  711. flexmock(module.borgmatic.actions.create).should_receive('collect_patterns').and_return(
  712. flexmock()
  713. )
  714. flexmock(module.borgmatic.actions.create).should_receive('process_patterns').and_return(
  715. [Pattern('foo'), Pattern('bar')]
  716. )
  717. flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
  718. dry_run=True,
  719. repository_path='repo',
  720. config=object,
  721. patterns=[Pattern('foo'), Pattern('bar')],
  722. local_borg_version=object,
  723. global_arguments=object,
  724. borgmatic_runtime_directory='/run/borgmatic',
  725. local_path=object,
  726. remote_path=object,
  727. list_files=True,
  728. stream_processes=True,
  729. ).and_return((('borg', 'create'), ('repo::archive',), flexmock()))
  730. flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return(
  731. flexmock()
  732. )
  733. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  734. '/working/dir'
  735. )
  736. flexmock(module.borgmatic.execute).should_receive(
  737. 'execute_command_and_capture_output'
  738. ).and_return(
  739. 'warning: stuff\n- foo\n+ bar\n? /nope',
  740. )
  741. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/foo').and_return(True)
  742. flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/bar').and_return(True)
  743. assert module.collect_spot_check_source_paths(
  744. repository={'path': 'repo'},
  745. config={'working_directory': '/working/dir'},
  746. local_borg_version=flexmock(),
  747. global_arguments=flexmock(),
  748. local_path=flexmock(),
  749. remote_path=flexmock(),
  750. borgmatic_runtime_directory='/run/borgmatic',
  751. ) == ('foo', 'bar')
  752. def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
  753. flexmock(module.random).should_receive('sample').replace_with(
  754. lambda population, count: population[:count]
  755. )
  756. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  757. None,
  758. )
  759. flexmock(module.os.path).should_receive('exists').and_return(True)
  760. flexmock(module.borgmatic.execute).should_receive(
  761. 'execute_command_and_capture_output'
  762. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  763. 'hash1 /foo\nhash2 /bar'
  764. )
  765. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  766. ['hash1 foo', 'nothash2 bar']
  767. )
  768. assert module.compare_spot_check_hashes(
  769. repository={'path': 'repo'},
  770. archive='archive',
  771. config={
  772. 'checks': [
  773. {
  774. 'name': 'archives',
  775. 'frequency': '2 weeks',
  776. },
  777. {
  778. 'name': 'spot',
  779. 'data_sample_percentage': 50,
  780. },
  781. ]
  782. },
  783. local_borg_version=flexmock(),
  784. global_arguments=flexmock(),
  785. local_path=flexmock(),
  786. remote_path=flexmock(),
  787. log_prefix='repo',
  788. source_paths=('/foo', '/bar', '/baz', '/quux'),
  789. ) == ('/bar',)
  790. def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes():
  791. flexmock(module.random).should_receive('sample').replace_with(
  792. lambda population, count: population[:count]
  793. )
  794. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  795. None,
  796. )
  797. flexmock(module.os.path).should_receive('exists').and_return(True)
  798. flexmock(module.borgmatic.execute).should_receive(
  799. 'execute_command_and_capture_output'
  800. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return(
  801. 'hash1 foo\nhash2 bar'
  802. )
  803. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  804. ['hash1 foo', 'nothash2 bar']
  805. )
  806. assert module.compare_spot_check_hashes(
  807. repository={'path': 'repo'},
  808. archive='archive',
  809. config={
  810. 'checks': [
  811. {
  812. 'name': 'archives',
  813. 'frequency': '2 weeks',
  814. },
  815. {
  816. 'name': 'spot',
  817. 'data_sample_percentage': 50,
  818. },
  819. ]
  820. },
  821. local_borg_version=flexmock(),
  822. global_arguments=flexmock(),
  823. local_path=flexmock(),
  824. remote_path=flexmock(),
  825. log_prefix='repo',
  826. source_paths=('foo', 'bar', 'baz', 'quux'),
  827. ) == ('bar',)
  828. def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
  829. flexmock(module.random).should_receive('sample').replace_with(
  830. lambda population, count: population[:count]
  831. )
  832. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  833. None,
  834. )
  835. flexmock(module.os.path).should_receive('exists').and_return(True)
  836. flexmock(module.borgmatic.execute).should_receive(
  837. 'execute_command_and_capture_output'
  838. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  839. 'hash1 /foo\nhash2 /bar'
  840. )
  841. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  842. ['nothash1 foo', 'nothash2 bar']
  843. )
  844. assert module.compare_spot_check_hashes(
  845. repository={'path': 'repo'},
  846. archive='archive',
  847. config={
  848. 'checks': [
  849. {
  850. 'name': 'archives',
  851. 'frequency': '2 weeks',
  852. },
  853. {
  854. 'name': 'spot',
  855. 'data_sample_percentage': 1000,
  856. },
  857. ]
  858. },
  859. local_borg_version=flexmock(),
  860. global_arguments=flexmock(),
  861. local_path=flexmock(),
  862. remote_path=flexmock(),
  863. log_prefix='repo',
  864. source_paths=('/foo', '/bar'),
  865. ) == ('/foo', '/bar')
  866. def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
  867. flexmock(module.random).should_receive('sample').replace_with(
  868. lambda population, count: population[:count]
  869. )
  870. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  871. None,
  872. )
  873. flexmock(module.os.path).should_receive('exists').and_return(True)
  874. flexmock(module.borgmatic.execute).should_receive(
  875. 'execute_command_and_capture_output'
  876. ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  877. 'hash1 /foo\nhash2 /bar'
  878. )
  879. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  880. ['hash1 foo', 'nothash2 bar']
  881. )
  882. assert module.compare_spot_check_hashes(
  883. repository={'path': 'repo'},
  884. archive='archive',
  885. config={
  886. 'checks': [
  887. {
  888. 'name': 'spot',
  889. 'data_sample_percentage': 50,
  890. 'xxh64sum_command': '/usr/local/bin/xxh64sum',
  891. },
  892. ]
  893. },
  894. local_borg_version=flexmock(),
  895. global_arguments=flexmock(),
  896. local_path=flexmock(),
  897. remote_path=flexmock(),
  898. log_prefix='repo',
  899. source_paths=('/foo', '/bar', '/baz', '/quux'),
  900. ) == ('/bar',)
  901. def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_matching():
  902. flexmock(module.random).should_receive('sample').replace_with(
  903. lambda population, count: population[:count]
  904. )
  905. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  906. None,
  907. )
  908. flexmock(module.os.path).should_receive('exists').and_return(True)
  909. flexmock(module.borgmatic.execute).should_receive(
  910. 'execute_command_and_capture_output'
  911. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  912. 'hash1 /foo\nhash2 /bar'
  913. )
  914. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  915. ['hash1 foo']
  916. )
  917. assert module.compare_spot_check_hashes(
  918. repository={'path': 'repo'},
  919. archive='archive',
  920. config={
  921. 'checks': [
  922. {
  923. 'name': 'spot',
  924. 'data_sample_percentage': 50,
  925. },
  926. ]
  927. },
  928. local_borg_version=flexmock(),
  929. global_arguments=flexmock(),
  930. local_path=flexmock(),
  931. remote_path=flexmock(),
  932. log_prefix='repo',
  933. source_paths=('/foo', '/bar', '/baz', '/quux'),
  934. ) == ('/bar',)
  935. def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching():
  936. flexmock(module.random).should_receive('sample').replace_with(
  937. lambda population, count: population[:count]
  938. )
  939. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  940. None,
  941. )
  942. flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True)
  943. flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False)
  944. flexmock(module.borgmatic.execute).should_receive(
  945. 'execute_command_and_capture_output'
  946. ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
  947. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  948. ['hash1 foo', 'hash2 bar']
  949. )
  950. assert module.compare_spot_check_hashes(
  951. repository={'path': 'repo'},
  952. archive='archive',
  953. config={
  954. 'checks': [
  955. {
  956. 'name': 'spot',
  957. 'data_sample_percentage': 50,
  958. },
  959. ]
  960. },
  961. local_borg_version=flexmock(),
  962. global_arguments=flexmock(),
  963. local_path=flexmock(),
  964. remote_path=flexmock(),
  965. log_prefix='repo',
  966. source_paths=('/foo', '/bar', '/baz', '/quux'),
  967. ) == ('/bar',)
  968. def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in_chunks():
  969. flexmock(module).SAMPLE_PATHS_SUBSET_COUNT = 2
  970. flexmock(module.random).should_receive('sample').replace_with(
  971. lambda population, count: population[:count]
  972. )
  973. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  974. None,
  975. )
  976. flexmock(module.os.path).should_receive('exists').and_return(True)
  977. flexmock(module.borgmatic.execute).should_receive(
  978. 'execute_command_and_capture_output'
  979. ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
  980. 'hash1 /foo\nhash2 /bar'
  981. )
  982. flexmock(module.borgmatic.execute).should_receive(
  983. 'execute_command_and_capture_output'
  984. ).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_return(
  985. 'hash3 /baz\nhash4 /quux'
  986. )
  987. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  988. ['hash1 foo', 'hash2 bar']
  989. ).and_return(['hash3 baz', 'nothash4 quux'])
  990. assert module.compare_spot_check_hashes(
  991. repository={'path': 'repo'},
  992. archive='archive',
  993. config={
  994. 'checks': [
  995. {
  996. 'name': 'archives',
  997. 'frequency': '2 weeks',
  998. },
  999. {
  1000. 'name': 'spot',
  1001. 'data_sample_percentage': 100,
  1002. },
  1003. ]
  1004. },
  1005. local_borg_version=flexmock(),
  1006. global_arguments=flexmock(),
  1007. local_path=flexmock(),
  1008. remote_path=flexmock(),
  1009. log_prefix='repo',
  1010. source_paths=('/foo', '/bar', '/baz', '/quux'),
  1011. ) == ('/quux',)
  1012. def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths():
  1013. flexmock(module.random).should_receive('sample').replace_with(
  1014. lambda population, count: population[:count]
  1015. )
  1016. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  1017. '/working/dir',
  1018. )
  1019. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True)
  1020. flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True)
  1021. flexmock(module.borgmatic.execute).should_receive(
  1022. 'execute_command_and_capture_output'
  1023. ).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return(
  1024. 'hash1 foo\nhash2 bar'
  1025. )
  1026. flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
  1027. ['hash1 foo', 'nothash2 bar']
  1028. )
  1029. assert module.compare_spot_check_hashes(
  1030. repository={'path': 'repo'},
  1031. archive='archive',
  1032. config={
  1033. 'checks': [
  1034. {
  1035. 'name': 'archives',
  1036. 'frequency': '2 weeks',
  1037. },
  1038. {
  1039. 'name': 'spot',
  1040. 'data_sample_percentage': 50,
  1041. },
  1042. ],
  1043. 'working_directory': '/working/dir',
  1044. },
  1045. local_borg_version=flexmock(),
  1046. global_arguments=flexmock(),
  1047. local_path=flexmock(),
  1048. remote_path=flexmock(),
  1049. log_prefix='repo',
  1050. source_paths=('foo', 'bar', 'baz', 'quux'),
  1051. ) == ('bar',)
  1052. def test_spot_check_without_spot_configuration_errors():
  1053. with pytest.raises(ValueError):
  1054. module.spot_check(
  1055. repository={'path': 'repo'},
  1056. config={
  1057. 'checks': [
  1058. {
  1059. 'name': 'archives',
  1060. },
  1061. ]
  1062. },
  1063. local_borg_version=flexmock(),
  1064. global_arguments=flexmock(),
  1065. local_path=flexmock(),
  1066. remote_path=flexmock(),
  1067. borgmatic_runtime_directory='/run/borgmatic',
  1068. )
  1069. def test_spot_check_without_any_configuration_errors():
  1070. with pytest.raises(ValueError):
  1071. module.spot_check(
  1072. repository={'path': 'repo'},
  1073. config={},
  1074. local_borg_version=flexmock(),
  1075. global_arguments=flexmock(),
  1076. local_path=flexmock(),
  1077. remote_path=flexmock(),
  1078. borgmatic_runtime_directory='/run/borgmatic',
  1079. )
  1080. def test_spot_check_data_tolerance_percentage_greater_than_data_sample_percentage_errors():
  1081. with pytest.raises(ValueError):
  1082. module.spot_check(
  1083. repository={'path': 'repo'},
  1084. config={
  1085. 'checks': [
  1086. {
  1087. 'name': 'spot',
  1088. 'data_tolerance_percentage': 7,
  1089. 'data_sample_percentage': 5,
  1090. },
  1091. ]
  1092. },
  1093. local_borg_version=flexmock(),
  1094. global_arguments=flexmock(),
  1095. local_path=flexmock(),
  1096. remote_path=flexmock(),
  1097. borgmatic_runtime_directory='/run/borgmatic',
  1098. )
  1099. def test_spot_check_with_count_delta_greater_than_count_tolerance_percentage_errors():
  1100. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1101. ('/foo', '/bar', '/baz', '/quux')
  1102. )
  1103. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1104. 'archive'
  1105. )
  1106. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(
  1107. ('/foo', '/bar')
  1108. ).once()
  1109. with pytest.raises(ValueError):
  1110. module.spot_check(
  1111. repository={'path': 'repo'},
  1112. config={
  1113. 'checks': [
  1114. {
  1115. 'name': 'spot',
  1116. 'count_tolerance_percentage': 1,
  1117. 'data_tolerance_percentage': 4,
  1118. 'data_sample_percentage': 5,
  1119. },
  1120. ]
  1121. },
  1122. local_borg_version=flexmock(),
  1123. global_arguments=flexmock(),
  1124. local_path=flexmock(),
  1125. remote_path=flexmock(),
  1126. borgmatic_runtime_directory='/run/borgmatic',
  1127. )
  1128. def test_spot_check_with_failing_percentage_greater_than_data_tolerance_percentage_errors():
  1129. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1130. ('/foo', '/bar', '/baz', '/quux')
  1131. )
  1132. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1133. 'archive'
  1134. )
  1135. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1136. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1137. ('/bar', '/baz', '/quux')
  1138. ).once()
  1139. with pytest.raises(ValueError):
  1140. module.spot_check(
  1141. repository={'path': 'repo'},
  1142. config={
  1143. 'checks': [
  1144. {
  1145. 'name': 'spot',
  1146. 'count_tolerance_percentage': 55,
  1147. 'data_tolerance_percentage': 4,
  1148. 'data_sample_percentage': 5,
  1149. },
  1150. ]
  1151. },
  1152. local_borg_version=flexmock(),
  1153. global_arguments=flexmock(),
  1154. local_path=flexmock(),
  1155. remote_path=flexmock(),
  1156. borgmatic_runtime_directory='/run/borgmatic',
  1157. )
  1158. def test_spot_check_with_high_enough_tolerances_does_not_raise():
  1159. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(
  1160. ('/foo', '/bar', '/baz', '/quux')
  1161. )
  1162. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1163. 'archive'
  1164. )
  1165. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1166. flexmock(module).should_receive('compare_spot_check_hashes').and_return(
  1167. ('/bar', '/baz', '/quux')
  1168. ).once()
  1169. module.spot_check(
  1170. repository={'path': 'repo'},
  1171. config={
  1172. 'checks': [
  1173. {
  1174. 'name': 'spot',
  1175. 'count_tolerance_percentage': 55,
  1176. 'data_tolerance_percentage': 80,
  1177. 'data_sample_percentage': 80,
  1178. },
  1179. ]
  1180. },
  1181. local_borg_version=flexmock(),
  1182. global_arguments=flexmock(),
  1183. local_path=flexmock(),
  1184. remote_path=flexmock(),
  1185. borgmatic_runtime_directory='/run/borgmatic',
  1186. )
  1187. def test_spot_check_without_any_source_paths_errors():
  1188. flexmock(module).should_receive('collect_spot_check_source_paths').and_return(())
  1189. flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
  1190. 'archive'
  1191. )
  1192. flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
  1193. flexmock(module).should_receive('compare_spot_check_hashes').never()
  1194. with pytest.raises(ValueError):
  1195. module.spot_check(
  1196. repository={'path': 'repo'},
  1197. config={
  1198. 'checks': [
  1199. {
  1200. 'name': 'spot',
  1201. 'count_tolerance_percentage': 10,
  1202. 'data_tolerance_percentage': 40,
  1203. 'data_sample_percentage': 50,
  1204. },
  1205. ]
  1206. },
  1207. local_borg_version=flexmock(),
  1208. global_arguments=flexmock(),
  1209. local_path=flexmock(),
  1210. remote_path=flexmock(),
  1211. borgmatic_runtime_directory='/run/borgmatic',
  1212. )
  1213. def test_run_check_checks_archives_for_configured_repository():
  1214. flexmock(module.logger).answer = lambda message: None
  1215. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1216. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1217. flexmock(module).should_receive('upgrade_check_times')
  1218. flexmock(module).should_receive('parse_checks')
  1219. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1220. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1221. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1222. {'repository', 'archives'}
  1223. )
  1224. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1225. flexmock(module).should_receive('make_check_time_path')
  1226. flexmock(module).should_receive('write_check_time')
  1227. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1228. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1229. check_arguments = flexmock(
  1230. repository=None,
  1231. progress=flexmock(),
  1232. repair=flexmock(),
  1233. only_checks=flexmock(),
  1234. force=flexmock(),
  1235. )
  1236. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1237. module.run_check(
  1238. config_filename='test.yaml',
  1239. repository={'path': 'repo'},
  1240. config={'repositories': ['repo']},
  1241. hook_context={},
  1242. local_borg_version=None,
  1243. check_arguments=check_arguments,
  1244. global_arguments=global_arguments,
  1245. local_path=None,
  1246. remote_path=None,
  1247. )
  1248. def test_run_check_runs_configured_extract_check():
  1249. flexmock(module.logger).answer = lambda message: None
  1250. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1251. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1252. flexmock(module).should_receive('upgrade_check_times')
  1253. flexmock(module).should_receive('parse_checks')
  1254. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1255. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1256. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'extract'})
  1257. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1258. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').once()
  1259. flexmock(module).should_receive('make_check_time_path')
  1260. flexmock(module).should_receive('write_check_time')
  1261. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1262. check_arguments = flexmock(
  1263. repository=None,
  1264. progress=flexmock(),
  1265. repair=flexmock(),
  1266. only_checks=flexmock(),
  1267. force=flexmock(),
  1268. )
  1269. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1270. module.run_check(
  1271. config_filename='test.yaml',
  1272. repository={'path': 'repo'},
  1273. config={'repositories': ['repo']},
  1274. hook_context={},
  1275. local_borg_version=None,
  1276. check_arguments=check_arguments,
  1277. global_arguments=global_arguments,
  1278. local_path=None,
  1279. remote_path=None,
  1280. )
  1281. def test_run_check_runs_configured_spot_check():
  1282. flexmock(module.logger).answer = lambda message: None
  1283. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1284. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1285. flexmock(module).should_receive('upgrade_check_times')
  1286. flexmock(module).should_receive('parse_checks')
  1287. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1288. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1289. flexmock(module).should_receive('filter_checks_on_frequency').and_return({'spot'})
  1290. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1291. flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
  1292. flexmock()
  1293. )
  1294. flexmock(module.borgmatic.actions.check).should_receive('spot_check').once()
  1295. flexmock(module).should_receive('make_check_time_path')
  1296. flexmock(module).should_receive('write_check_time')
  1297. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1298. check_arguments = flexmock(
  1299. repository=None,
  1300. progress=flexmock(),
  1301. repair=flexmock(),
  1302. only_checks=flexmock(),
  1303. force=flexmock(),
  1304. )
  1305. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1306. module.run_check(
  1307. config_filename='test.yaml',
  1308. repository={'path': 'repo'},
  1309. config={'repositories': ['repo']},
  1310. hook_context={},
  1311. local_borg_version=None,
  1312. check_arguments=check_arguments,
  1313. global_arguments=global_arguments,
  1314. local_path=None,
  1315. remote_path=None,
  1316. )
  1317. def test_run_check_without_checks_runs_nothing_except_hooks():
  1318. flexmock(module.logger).answer = lambda message: None
  1319. flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
  1320. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1321. flexmock(module).should_receive('upgrade_check_times')
  1322. flexmock(module).should_receive('parse_checks')
  1323. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1324. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1325. flexmock(module).should_receive('filter_checks_on_frequency').and_return({})
  1326. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1327. flexmock(module).should_receive('make_check_time_path')
  1328. flexmock(module).should_receive('write_check_time').never()
  1329. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1330. flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
  1331. check_arguments = flexmock(
  1332. repository=None,
  1333. progress=flexmock(),
  1334. repair=flexmock(),
  1335. only_checks=flexmock(),
  1336. force=flexmock(),
  1337. )
  1338. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1339. module.run_check(
  1340. config_filename='test.yaml',
  1341. repository={'path': 'repo'},
  1342. config={'repositories': ['repo']},
  1343. hook_context={},
  1344. local_borg_version=None,
  1345. check_arguments=check_arguments,
  1346. global_arguments=global_arguments,
  1347. local_path=None,
  1348. remote_path=None,
  1349. )
  1350. def test_run_check_checks_archives_in_selected_repository():
  1351. flexmock(module.logger).answer = lambda message: None
  1352. flexmock(module.borgmatic.config.validate).should_receive(
  1353. 'repositories_match'
  1354. ).once().and_return(True)
  1355. flexmock(module.borgmatic.borg.check).should_receive('get_repository_id').and_return(flexmock())
  1356. flexmock(module).should_receive('upgrade_check_times')
  1357. flexmock(module).should_receive('parse_checks')
  1358. flexmock(module.borgmatic.borg.check).should_receive('make_archive_filter_flags').and_return(())
  1359. flexmock(module).should_receive('make_archives_check_id').and_return(None)
  1360. flexmock(module).should_receive('filter_checks_on_frequency').and_return(
  1361. {'repository', 'archives'}
  1362. )
  1363. flexmock(module.borgmatic.borg.check).should_receive('check_archives').once()
  1364. flexmock(module).should_receive('make_check_time_path')
  1365. flexmock(module).should_receive('write_check_time')
  1366. flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never()
  1367. check_arguments = flexmock(
  1368. repository=flexmock(),
  1369. progress=flexmock(),
  1370. repair=flexmock(),
  1371. only_checks=flexmock(),
  1372. force=flexmock(),
  1373. )
  1374. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1375. module.run_check(
  1376. config_filename='test.yaml',
  1377. repository={'path': 'repo'},
  1378. config={'repositories': ['repo']},
  1379. hook_context={},
  1380. local_borg_version=None,
  1381. check_arguments=check_arguments,
  1382. global_arguments=global_arguments,
  1383. local_path=None,
  1384. remote_path=None,
  1385. )
  1386. def test_run_check_bails_if_repository_does_not_match():
  1387. flexmock(module.logger).answer = lambda message: None
  1388. flexmock(module.borgmatic.config.validate).should_receive(
  1389. 'repositories_match'
  1390. ).once().and_return(False)
  1391. flexmock(module.borgmatic.borg.check).should_receive('check_archives').never()
  1392. check_arguments = flexmock(
  1393. repository=flexmock(),
  1394. progress=flexmock(),
  1395. repair=flexmock(),
  1396. only_checks=flexmock(),
  1397. force=flexmock(),
  1398. )
  1399. global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
  1400. module.run_check(
  1401. config_filename='test.yaml',
  1402. repository={'path': 'repo'},
  1403. config={'repositories': ['repo']},
  1404. hook_context={},
  1405. local_borg_version=None,
  1406. check_arguments=check_arguments,
  1407. global_arguments=global_arguments,
  1408. local_path=None,
  1409. remote_path=None,
  1410. )