test_borgmatic.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. import logging
  2. import subprocess
  3. import time
  4. from flexmock import flexmock
  5. import borgmatic.hooks.command
  6. from borgmatic.commands import borgmatic as module
  7. def test_run_configuration_runs_actions_for_each_repository():
  8. flexmock(module.borg_environment).should_receive('initialize')
  9. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  10. expected_results = [flexmock(), flexmock()]
  11. flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return(
  12. expected_results[1:]
  13. )
  14. config = {'location': {'repositories': ['foo', 'bar']}}
  15. arguments = {'global': flexmock(monitoring_verbosity=1)}
  16. results = list(module.run_configuration('test.yaml', config, arguments))
  17. assert results == expected_results
  18. def test_run_configuration_with_invalid_borg_version_errors():
  19. flexmock(module.borg_environment).should_receive('initialize')
  20. flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError)
  21. flexmock(module.command).should_receive('execute_hook').never()
  22. flexmock(module.dispatch).should_receive('call_hooks').never()
  23. flexmock(module).should_receive('run_actions').never()
  24. config = {'location': {'repositories': ['foo']}}
  25. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()}
  26. list(module.run_configuration('test.yaml', config, arguments))
  27. def test_run_configuration_logs_monitor_start_error():
  28. flexmock(module.borg_environment).should_receive('initialize')
  29. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  30. flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return(
  31. None
  32. ).and_return(None)
  33. expected_results = [flexmock()]
  34. flexmock(module).should_receive('log_error_records').and_return(expected_results)
  35. flexmock(module).should_receive('run_actions').never()
  36. config = {'location': {'repositories': ['foo']}}
  37. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  38. results = list(module.run_configuration('test.yaml', config, arguments))
  39. assert results == expected_results
  40. def test_run_configuration_bails_for_monitor_start_soft_failure():
  41. flexmock(module.borg_environment).should_receive('initialize')
  42. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  43. error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
  44. flexmock(module.dispatch).should_receive('call_hooks').and_raise(error)
  45. flexmock(module).should_receive('log_error_records').never()
  46. flexmock(module).should_receive('run_actions').never()
  47. config = {'location': {'repositories': ['foo']}}
  48. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  49. results = list(module.run_configuration('test.yaml', config, arguments))
  50. assert results == []
  51. def test_run_configuration_logs_actions_error():
  52. flexmock(module.borg_environment).should_receive('initialize')
  53. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  54. flexmock(module.command).should_receive('execute_hook')
  55. flexmock(module.dispatch).should_receive('call_hooks')
  56. expected_results = [flexmock()]
  57. flexmock(module).should_receive('log_error_records').and_return(expected_results)
  58. flexmock(module).should_receive('run_actions').and_raise(OSError)
  59. config = {'location': {'repositories': ['foo']}}
  60. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)}
  61. results = list(module.run_configuration('test.yaml', config, arguments))
  62. assert results == expected_results
  63. def test_run_configuration_bails_for_actions_soft_failure():
  64. flexmock(module.borg_environment).should_receive('initialize')
  65. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  66. flexmock(module.dispatch).should_receive('call_hooks')
  67. error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
  68. flexmock(module).should_receive('run_actions').and_raise(error)
  69. flexmock(module).should_receive('log_error_records').never()
  70. flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
  71. config = {'location': {'repositories': ['foo']}}
  72. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  73. results = list(module.run_configuration('test.yaml', config, arguments))
  74. assert results == []
  75. def test_run_configuration_logs_monitor_finish_error():
  76. flexmock(module.borg_environment).should_receive('initialize')
  77. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  78. flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
  79. None
  80. ).and_raise(OSError)
  81. expected_results = [flexmock()]
  82. flexmock(module).should_receive('log_error_records').and_return(expected_results)
  83. flexmock(module).should_receive('run_actions').and_return([])
  84. config = {'location': {'repositories': ['foo']}}
  85. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  86. results = list(module.run_configuration('test.yaml', config, arguments))
  87. assert results == expected_results
  88. def test_run_configuration_bails_for_monitor_finish_soft_failure():
  89. flexmock(module.borg_environment).should_receive('initialize')
  90. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  91. error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
  92. flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return(
  93. None
  94. ).and_raise(error)
  95. flexmock(module).should_receive('log_error_records').never()
  96. flexmock(module).should_receive('run_actions').and_return([])
  97. flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
  98. config = {'location': {'repositories': ['foo']}}
  99. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  100. results = list(module.run_configuration('test.yaml', config, arguments))
  101. assert results == []
  102. def test_run_configuration_logs_on_error_hook_error():
  103. flexmock(module.borg_environment).should_receive('initialize')
  104. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  105. flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
  106. expected_results = [flexmock(), flexmock()]
  107. flexmock(module).should_receive('log_error_records').and_return(
  108. expected_results[:1]
  109. ).and_return(expected_results[1:])
  110. flexmock(module).should_receive('run_actions').and_raise(OSError)
  111. config = {'location': {'repositories': ['foo']}}
  112. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  113. results = list(module.run_configuration('test.yaml', config, arguments))
  114. assert results == expected_results
  115. def test_run_configuration_bails_for_on_error_hook_soft_failure():
  116. flexmock(module.borg_environment).should_receive('initialize')
  117. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  118. error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
  119. flexmock(module.command).should_receive('execute_hook').and_raise(error)
  120. expected_results = [flexmock()]
  121. flexmock(module).should_receive('log_error_records').and_return(expected_results)
  122. flexmock(module).should_receive('run_actions').and_raise(OSError)
  123. config = {'location': {'repositories': ['foo']}}
  124. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  125. results = list(module.run_configuration('test.yaml', config, arguments))
  126. assert results == expected_results
  127. def test_run_configuration_retries_soft_error():
  128. # Run action first fails, second passes
  129. flexmock(module.borg_environment).should_receive('initialize')
  130. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  131. flexmock(module.command).should_receive('execute_hook')
  132. flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([])
  133. flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once()
  134. config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
  135. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  136. results = list(module.run_configuration('test.yaml', config, arguments))
  137. assert results == []
  138. def test_run_configuration_retries_hard_error():
  139. # Run action fails twice
  140. flexmock(module.borg_environment).should_receive('initialize')
  141. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  142. flexmock(module.command).should_receive('execute_hook')
  143. flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
  144. flexmock(module).should_receive('log_error_records').with_args(
  145. 'foo: Error running actions for repository',
  146. OSError,
  147. levelno=logging.WARNING,
  148. log_command_error_output=True,
  149. ).and_return([flexmock()])
  150. error_logs = [flexmock()]
  151. flexmock(module).should_receive('log_error_records').with_args(
  152. 'foo: Error running actions for repository', OSError,
  153. ).and_return(error_logs)
  154. config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
  155. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  156. results = list(module.run_configuration('test.yaml', config, arguments))
  157. assert results == error_logs
  158. def test_run_repos_ordered():
  159. flexmock(module.borg_environment).should_receive('initialize')
  160. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  161. flexmock(module.command).should_receive('execute_hook')
  162. flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
  163. expected_results = [flexmock(), flexmock()]
  164. flexmock(module).should_receive('log_error_records').with_args(
  165. 'foo: Error running actions for repository', OSError
  166. ).and_return(expected_results[:1]).ordered()
  167. flexmock(module).should_receive('log_error_records').with_args(
  168. 'bar: Error running actions for repository', OSError
  169. ).and_return(expected_results[1:]).ordered()
  170. config = {'location': {'repositories': ['foo', 'bar']}}
  171. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  172. results = list(module.run_configuration('test.yaml', config, arguments))
  173. assert results == expected_results
  174. def test_run_configuration_retries_round_robbin():
  175. flexmock(module.borg_environment).should_receive('initialize')
  176. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  177. flexmock(module.command).should_receive('execute_hook')
  178. flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
  179. flexmock(module).should_receive('log_error_records').with_args(
  180. 'foo: Error running actions for repository',
  181. OSError,
  182. levelno=logging.WARNING,
  183. log_command_error_output=True,
  184. ).and_return([flexmock()]).ordered()
  185. flexmock(module).should_receive('log_error_records').with_args(
  186. 'bar: Error running actions for repository',
  187. OSError,
  188. levelno=logging.WARNING,
  189. log_command_error_output=True,
  190. ).and_return([flexmock()]).ordered()
  191. foo_error_logs = [flexmock()]
  192. flexmock(module).should_receive('log_error_records').with_args(
  193. 'foo: Error running actions for repository', OSError
  194. ).and_return(foo_error_logs).ordered()
  195. bar_error_logs = [flexmock()]
  196. flexmock(module).should_receive('log_error_records').with_args(
  197. 'bar: Error running actions for repository', OSError
  198. ).and_return(bar_error_logs).ordered()
  199. config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
  200. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  201. results = list(module.run_configuration('test.yaml', config, arguments))
  202. assert results == foo_error_logs + bar_error_logs
  203. def test_run_configuration_retries_one_passes():
  204. flexmock(module.borg_environment).should_receive('initialize')
  205. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  206. flexmock(module.command).should_receive('execute_hook')
  207. flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
  208. []
  209. ).and_raise(OSError).times(4)
  210. flexmock(module).should_receive('log_error_records').with_args(
  211. 'foo: Error running actions for repository',
  212. OSError,
  213. levelno=logging.WARNING,
  214. log_command_error_output=True,
  215. ).and_return([flexmock()]).ordered()
  216. flexmock(module).should_receive('log_error_records').with_args(
  217. 'bar: Error running actions for repository',
  218. OSError,
  219. levelno=logging.WARNING,
  220. log_command_error_output=True,
  221. ).and_return(flexmock()).ordered()
  222. error_logs = [flexmock()]
  223. flexmock(module).should_receive('log_error_records').with_args(
  224. 'bar: Error running actions for repository', OSError
  225. ).and_return(error_logs).ordered()
  226. config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
  227. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  228. results = list(module.run_configuration('test.yaml', config, arguments))
  229. assert results == error_logs
  230. def test_run_configuration_retry_wait():
  231. flexmock(module.borg_environment).should_receive('initialize')
  232. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  233. flexmock(module.command).should_receive('execute_hook')
  234. flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
  235. flexmock(module).should_receive('log_error_records').with_args(
  236. 'foo: Error running actions for repository',
  237. OSError,
  238. levelno=logging.WARNING,
  239. log_command_error_output=True,
  240. ).and_return([flexmock()]).ordered()
  241. flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
  242. flexmock(module).should_receive('log_error_records').with_args(
  243. 'foo: Error running actions for repository',
  244. OSError,
  245. levelno=logging.WARNING,
  246. log_command_error_output=True,
  247. ).and_return([flexmock()]).ordered()
  248. flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
  249. flexmock(module).should_receive('log_error_records').with_args(
  250. 'foo: Error running actions for repository',
  251. OSError,
  252. levelno=logging.WARNING,
  253. log_command_error_output=True,
  254. ).and_return([flexmock()]).ordered()
  255. flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
  256. error_logs = [flexmock()]
  257. flexmock(module).should_receive('log_error_records').with_args(
  258. 'foo: Error running actions for repository', OSError
  259. ).and_return(error_logs).ordered()
  260. config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
  261. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  262. results = list(module.run_configuration('test.yaml', config, arguments))
  263. assert results == error_logs
  264. def test_run_configuration_retries_timeout_multiple_repos():
  265. flexmock(module.borg_environment).should_receive('initialize')
  266. flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
  267. flexmock(module.command).should_receive('execute_hook')
  268. flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
  269. []
  270. ).and_raise(OSError).times(4)
  271. flexmock(module).should_receive('log_error_records').with_args(
  272. 'foo: Error running actions for repository',
  273. OSError,
  274. levelno=logging.WARNING,
  275. log_command_error_output=True,
  276. ).and_return([flexmock()]).ordered()
  277. flexmock(module).should_receive('log_error_records').with_args(
  278. 'bar: Error running actions for repository',
  279. OSError,
  280. levelno=logging.WARNING,
  281. log_command_error_output=True,
  282. ).and_return([flexmock()]).ordered()
  283. # Sleep before retrying foo (and passing)
  284. flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
  285. # Sleep before retrying bar (and failing)
  286. flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
  287. error_logs = [flexmock()]
  288. flexmock(module).should_receive('log_error_records').with_args(
  289. 'bar: Error running actions for repository', OSError
  290. ).and_return(error_logs).ordered()
  291. config = {
  292. 'location': {'repositories': ['foo', 'bar']},
  293. 'storage': {'retries': 1, 'retry_wait': 10},
  294. }
  295. arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
  296. results = list(module.run_configuration('test.yaml', config, arguments))
  297. assert results == error_logs
  298. def test_run_actions_does_not_raise_for_init_action():
  299. flexmock(module.borg_init).should_receive('initialize_repository')
  300. arguments = {
  301. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  302. 'init': flexmock(
  303. encryption_mode=flexmock(), append_only=flexmock(), storage_quota=flexmock()
  304. ),
  305. }
  306. list(
  307. module.run_actions(
  308. arguments=arguments,
  309. config_filename='test.yaml',
  310. location={'repositories': ['repo']},
  311. storage={},
  312. retention={},
  313. consistency={},
  314. hooks={},
  315. local_path=None,
  316. remote_path=None,
  317. local_borg_version=None,
  318. repository_path='repo',
  319. )
  320. )
  321. def test_run_actions_calls_hooks_for_prune_action():
  322. flexmock(module.borg_prune).should_receive('prune_archives')
  323. flexmock(module.command).should_receive('execute_hook').twice()
  324. arguments = {
  325. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  326. 'prune': flexmock(stats=flexmock(), files=flexmock()),
  327. }
  328. list(
  329. module.run_actions(
  330. arguments=arguments,
  331. config_filename='test.yaml',
  332. location={'repositories': ['repo']},
  333. storage={},
  334. retention={},
  335. consistency={},
  336. hooks={},
  337. local_path=None,
  338. remote_path=None,
  339. local_borg_version=None,
  340. repository_path='repo',
  341. )
  342. )
  343. def test_run_actions_calls_hooks_for_compact_action():
  344. flexmock(module.borg_feature).should_receive('available').and_return(True)
  345. flexmock(module.borg_compact).should_receive('compact_segments')
  346. flexmock(module.command).should_receive('execute_hook').twice()
  347. arguments = {
  348. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  349. 'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()),
  350. }
  351. list(
  352. module.run_actions(
  353. arguments=arguments,
  354. config_filename='test.yaml',
  355. location={'repositories': ['repo']},
  356. storage={},
  357. retention={},
  358. consistency={},
  359. hooks={},
  360. local_path=None,
  361. remote_path=None,
  362. local_borg_version=None,
  363. repository_path='repo',
  364. )
  365. )
  366. def test_run_actions_executes_and_calls_hooks_for_create_action():
  367. flexmock(module.borg_create).should_receive('create_archive')
  368. flexmock(module.command).should_receive('execute_hook').twice()
  369. flexmock(module.dispatch).should_receive('call_hooks').and_return({}).times(3)
  370. arguments = {
  371. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  372. 'create': flexmock(
  373. progress=flexmock(), stats=flexmock(), json=flexmock(), files=flexmock()
  374. ),
  375. }
  376. list(
  377. module.run_actions(
  378. arguments=arguments,
  379. config_filename='test.yaml',
  380. location={'repositories': ['repo']},
  381. storage={},
  382. retention={},
  383. consistency={},
  384. hooks={},
  385. local_path=None,
  386. remote_path=None,
  387. local_borg_version=None,
  388. repository_path='repo',
  389. )
  390. )
  391. def test_run_actions_calls_hooks_for_check_action():
  392. flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
  393. flexmock(module.borg_check).should_receive('check_archives')
  394. flexmock(module.command).should_receive('execute_hook').twice()
  395. arguments = {
  396. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  397. 'check': flexmock(
  398. progress=flexmock(), repair=flexmock(), only=flexmock(), force=flexmock()
  399. ),
  400. }
  401. list(
  402. module.run_actions(
  403. arguments=arguments,
  404. config_filename='test.yaml',
  405. location={'repositories': ['repo']},
  406. storage={},
  407. retention={},
  408. consistency={},
  409. hooks={},
  410. local_path=None,
  411. remote_path=None,
  412. local_borg_version=None,
  413. repository_path='repo',
  414. )
  415. )
  416. def test_run_actions_calls_hooks_for_extract_action():
  417. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  418. flexmock(module.borg_extract).should_receive('extract_archive')
  419. flexmock(module.command).should_receive('execute_hook').twice()
  420. arguments = {
  421. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  422. 'extract': flexmock(
  423. paths=flexmock(),
  424. progress=flexmock(),
  425. destination=flexmock(),
  426. strip_components=flexmock(),
  427. archive=flexmock(),
  428. repository='repo',
  429. ),
  430. }
  431. list(
  432. module.run_actions(
  433. arguments=arguments,
  434. config_filename='test.yaml',
  435. location={'repositories': ['repo']},
  436. storage={},
  437. retention={},
  438. consistency={},
  439. hooks={},
  440. local_path=None,
  441. remote_path=None,
  442. local_borg_version=None,
  443. repository_path='repo',
  444. )
  445. )
  446. def test_run_actions_does_not_raise_for_export_tar_action():
  447. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  448. flexmock(module.borg_export_tar).should_receive('export_tar_archive')
  449. arguments = {
  450. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  451. 'export-tar': flexmock(
  452. repository=flexmock(),
  453. archive=flexmock(),
  454. paths=flexmock(),
  455. destination=flexmock(),
  456. tar_filter=flexmock(),
  457. files=flexmock(),
  458. strip_components=flexmock(),
  459. ),
  460. }
  461. list(
  462. module.run_actions(
  463. arguments=arguments,
  464. config_filename='test.yaml',
  465. location={'repositories': ['repo']},
  466. storage={},
  467. retention={},
  468. consistency={},
  469. hooks={},
  470. local_path=None,
  471. remote_path=None,
  472. local_borg_version=None,
  473. repository_path='repo',
  474. )
  475. )
  476. def test_run_actions_does_not_raise_for_mount_action():
  477. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  478. flexmock(module.borg_mount).should_receive('mount_archive')
  479. arguments = {
  480. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  481. 'mount': flexmock(
  482. repository=flexmock(),
  483. archive=flexmock(),
  484. mount_point=flexmock(),
  485. paths=flexmock(),
  486. foreground=flexmock(),
  487. options=flexmock(),
  488. ),
  489. }
  490. list(
  491. module.run_actions(
  492. arguments=arguments,
  493. config_filename='test.yaml',
  494. location={'repositories': ['repo']},
  495. storage={},
  496. retention={},
  497. consistency={},
  498. hooks={},
  499. local_path=None,
  500. remote_path=None,
  501. local_borg_version=None,
  502. repository_path='repo',
  503. )
  504. )
  505. def test_run_actions_does_not_raise_for_list_action():
  506. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  507. flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
  508. flexmock(module.borg_list).should_receive('list_archives')
  509. arguments = {
  510. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  511. 'list': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
  512. }
  513. list(
  514. module.run_actions(
  515. arguments=arguments,
  516. config_filename='test.yaml',
  517. location={'repositories': ['repo']},
  518. storage={},
  519. retention={},
  520. consistency={},
  521. hooks={},
  522. local_path=None,
  523. remote_path=None,
  524. local_borg_version=None,
  525. repository_path='repo',
  526. )
  527. )
  528. def test_run_actions_does_not_raise_for_info_action():
  529. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  530. flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
  531. flexmock(module.borg_info).should_receive('display_archives_info')
  532. arguments = {
  533. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  534. 'info': flexmock(repository=flexmock(), archive=flexmock(), json=flexmock()),
  535. }
  536. list(
  537. module.run_actions(
  538. arguments=arguments,
  539. config_filename='test.yaml',
  540. location={'repositories': ['repo']},
  541. storage={},
  542. retention={},
  543. consistency={},
  544. hooks={},
  545. local_path=None,
  546. remote_path=None,
  547. local_borg_version=None,
  548. repository_path='repo',
  549. )
  550. )
  551. def test_run_actions_does_not_raise_for_borg_action():
  552. flexmock(module.validate).should_receive('repositories_match').and_return(True)
  553. flexmock(module.borg_list).should_receive('resolve_archive_name').and_return(flexmock())
  554. flexmock(module.borg_borg).should_receive('run_arbitrary_borg')
  555. arguments = {
  556. 'global': flexmock(monitoring_verbosity=1, dry_run=False),
  557. 'borg': flexmock(repository=flexmock(), archive=flexmock(), options=flexmock()),
  558. }
  559. list(
  560. module.run_actions(
  561. arguments=arguments,
  562. config_filename='test.yaml',
  563. location={'repositories': ['repo']},
  564. storage={},
  565. retention={},
  566. consistency={},
  567. hooks={},
  568. local_path=None,
  569. remote_path=None,
  570. local_borg_version=None,
  571. repository_path='repo',
  572. )
  573. )
  574. def test_load_configurations_collects_parsed_configurations():
  575. configuration = flexmock()
  576. other_configuration = flexmock()
  577. flexmock(module.validate).should_receive('parse_configuration').and_return(
  578. configuration
  579. ).and_return(other_configuration)
  580. configs, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml')))
  581. assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration}
  582. assert logs == []
  583. def test_load_configurations_logs_warning_for_permission_error():
  584. flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError)
  585. configs, logs = tuple(module.load_configurations(('test.yaml',)))
  586. assert configs == {}
  587. assert {log.levelno for log in logs} == {logging.WARNING}
  588. def test_load_configurations_logs_critical_for_parse_error():
  589. flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
  590. configs, logs = tuple(module.load_configurations(('test.yaml',)))
  591. assert configs == {}
  592. assert {log.levelno for log in logs} == {logging.CRITICAL}
  593. def test_log_record_does_not_raise():
  594. module.log_record(levelno=1, foo='bar', baz='quux')
  595. def test_log_record_with_suppress_does_not_raise():
  596. module.log_record(levelno=1, foo='bar', baz='quux', suppress_log=True)
  597. def test_log_error_records_generates_output_logs_for_message_only():
  598. flexmock(module).should_receive('log_record').replace_with(dict)
  599. logs = tuple(module.log_error_records('Error'))
  600. assert {log['levelno'] for log in logs} == {logging.CRITICAL}
  601. def test_log_error_records_generates_output_logs_for_called_process_error():
  602. flexmock(module).should_receive('log_record').replace_with(dict)
  603. flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING)
  604. logs = tuple(
  605. module.log_error_records('Error', subprocess.CalledProcessError(1, 'ls', 'error output'))
  606. )
  607. assert {log['levelno'] for log in logs} == {logging.CRITICAL}
  608. assert any(log for log in logs if 'error output' in str(log))
  609. def test_log_error_records_generates_logs_for_value_error():
  610. flexmock(module).should_receive('log_record').replace_with(dict)
  611. logs = tuple(module.log_error_records('Error', ValueError()))
  612. assert {log['levelno'] for log in logs} == {logging.CRITICAL}
  613. def test_log_error_records_generates_logs_for_os_error():
  614. flexmock(module).should_receive('log_record').replace_with(dict)
  615. logs = tuple(module.log_error_records('Error', OSError()))
  616. assert {log['levelno'] for log in logs} == {logging.CRITICAL}
  617. def test_log_error_records_generates_nothing_for_other_error():
  618. flexmock(module).should_receive('log_record').replace_with(dict)
  619. logs = tuple(module.log_error_records('Error', KeyError()))
  620. assert logs == ()
  621. def test_get_local_path_uses_configuration_value():
  622. assert module.get_local_path({'test.yaml': {'location': {'local_path': 'borg1'}}}) == 'borg1'
  623. def test_get_local_path_without_location_defaults_to_borg():
  624. assert module.get_local_path({'test.yaml': {}}) == 'borg'
  625. def test_get_local_path_without_local_path_defaults_to_borg():
  626. assert module.get_local_path({'test.yaml': {'location': {}}}) == 'borg'
  627. def test_collect_configuration_run_summary_logs_info_for_success():
  628. flexmock(module.command).should_receive('execute_hook').never()
  629. flexmock(module).should_receive('run_configuration').and_return([])
  630. arguments = {}
  631. logs = tuple(
  632. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  633. )
  634. assert {log.levelno for log in logs} == {logging.INFO}
  635. def test_collect_configuration_run_summary_executes_hooks_for_create():
  636. flexmock(module).should_receive('run_configuration').and_return([])
  637. arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)}
  638. logs = tuple(
  639. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  640. )
  641. assert {log.levelno for log in logs} == {logging.INFO}
  642. def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
  643. flexmock(module.validate).should_receive('guard_configuration_contains_repository')
  644. flexmock(module).should_receive('run_configuration').and_return([])
  645. arguments = {'extract': flexmock(repository='repo')}
  646. logs = tuple(
  647. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  648. )
  649. assert {log.levelno for log in logs} == {logging.INFO}
  650. def test_collect_configuration_run_summary_logs_extract_with_repository_error():
  651. flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
  652. ValueError
  653. )
  654. expected_logs = (flexmock(),)
  655. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  656. arguments = {'extract': flexmock(repository='repo')}
  657. logs = tuple(
  658. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  659. )
  660. assert logs == expected_logs
  661. def test_collect_configuration_run_summary_logs_info_for_success_with_mount():
  662. flexmock(module.validate).should_receive('guard_configuration_contains_repository')
  663. flexmock(module).should_receive('run_configuration').and_return([])
  664. arguments = {'mount': flexmock(repository='repo')}
  665. logs = tuple(
  666. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  667. )
  668. assert {log.levelno for log in logs} == {logging.INFO}
  669. def test_collect_configuration_run_summary_logs_mount_with_repository_error():
  670. flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
  671. ValueError
  672. )
  673. expected_logs = (flexmock(),)
  674. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  675. arguments = {'mount': flexmock(repository='repo')}
  676. logs = tuple(
  677. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  678. )
  679. assert logs == expected_logs
  680. def test_collect_configuration_run_summary_logs_missing_configs_error():
  681. arguments = {'global': flexmock(config_paths=[])}
  682. expected_logs = (flexmock(),)
  683. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  684. logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
  685. assert logs == expected_logs
  686. def test_collect_configuration_run_summary_logs_pre_hook_error():
  687. flexmock(module.command).should_receive('execute_hook').and_raise(ValueError)
  688. expected_logs = (flexmock(),)
  689. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  690. arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)}
  691. logs = tuple(
  692. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  693. )
  694. assert logs == expected_logs
  695. def test_collect_configuration_run_summary_logs_post_hook_error():
  696. flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError)
  697. flexmock(module).should_receive('run_configuration').and_return([])
  698. expected_logs = (flexmock(),)
  699. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  700. arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)}
  701. logs = tuple(
  702. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  703. )
  704. assert expected_logs[0] in logs
  705. def test_collect_configuration_run_summary_logs_for_list_with_archive_and_repository_error():
  706. flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
  707. ValueError
  708. )
  709. expected_logs = (flexmock(),)
  710. flexmock(module).should_receive('log_error_records').and_return(expected_logs)
  711. arguments = {'list': flexmock(repository='repo', archive='test')}
  712. logs = tuple(
  713. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  714. )
  715. assert logs == expected_logs
  716. def test_collect_configuration_run_summary_logs_info_for_success_with_list():
  717. flexmock(module).should_receive('run_configuration').and_return([])
  718. arguments = {'list': flexmock(repository='repo', archive=None)}
  719. logs = tuple(
  720. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  721. )
  722. assert {log.levelno for log in logs} == {logging.INFO}
  723. def test_collect_configuration_run_summary_logs_run_configuration_error():
  724. flexmock(module.validate).should_receive('guard_configuration_contains_repository')
  725. flexmock(module).should_receive('run_configuration').and_return(
  726. [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
  727. )
  728. flexmock(module).should_receive('log_error_records').and_return([])
  729. arguments = {}
  730. logs = tuple(
  731. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  732. )
  733. assert {log.levelno for log in logs} == {logging.CRITICAL}
  734. def test_collect_configuration_run_summary_logs_run_umount_error():
  735. flexmock(module.validate).should_receive('guard_configuration_contains_repository')
  736. flexmock(module).should_receive('run_configuration').and_return([])
  737. flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError)
  738. flexmock(module).should_receive('log_error_records').and_return(
  739. [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
  740. )
  741. arguments = {'umount': flexmock(mount_point='/mnt')}
  742. logs = tuple(
  743. module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
  744. )
  745. assert {log.levelno for log in logs} == {logging.INFO, logging.CRITICAL}
  746. def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
  747. flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
  748. ['baz']
  749. )
  750. stdout = flexmock()
  751. stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once()
  752. flexmock(module.sys).stdout = stdout
  753. arguments = {}
  754. tuple(
  755. module.collect_configuration_run_summary_logs(
  756. {'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments
  757. )
  758. )