test_postgresql.py 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650
  1. import logging
  2. import pytest
  3. from flexmock import flexmock
  4. from borgmatic.hooks.data_source import postgresql as module
  5. def test_make_environment_maps_options_to_environment():
  6. database = {
  7. 'name': 'foo',
  8. 'password': 'pass',
  9. 'ssl_mode': 'require',
  10. 'ssl_cert': 'cert.crt',
  11. 'ssl_key': 'key.key',
  12. 'ssl_root_cert': 'root.crt',
  13. 'ssl_crl': 'crl.crl',
  14. }
  15. expected = {
  16. 'USER': 'root',
  17. 'PGPASSWORD': 'pass',
  18. 'PGSSLMODE': 'require',
  19. 'PGSSLCERT': 'cert.crt',
  20. 'PGSSLKEY': 'key.key',
  21. 'PGSSLROOTCERT': 'root.crt',
  22. 'PGSSLCRL': 'crl.crl',
  23. }
  24. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  25. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  26. 'resolve_credential',
  27. ).replace_with(lambda value, config: value)
  28. assert module.make_environment(database, {}) == expected
  29. def test_make_environment_with_cli_password_sets_correct_password():
  30. database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
  31. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  32. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  33. 'resolve_credential',
  34. ).replace_with(lambda value, config: value)
  35. environment = module.make_environment(
  36. database,
  37. {},
  38. restore_connection_params={'password': 'clipassword'},
  39. )
  40. assert environment['PGPASSWORD'] == 'clipassword'
  41. def test_make_environment_without_cli_password_or_configured_password_does_not_set_password():
  42. database = {'name': 'foo'}
  43. environment = module.make_environment(
  44. database,
  45. {},
  46. restore_connection_params={'username': 'someone'},
  47. )
  48. assert 'PGPASSWORD' not in environment
  49. def test_make_environment_without_ssl_mode_does_not_set_ssl_mode():
  50. database = {'name': 'foo'}
  51. environment = module.make_environment(database, {})
  52. assert 'PGSSLMODE' not in environment
  53. def test_database_names_to_dump_passes_through_individual_database_name():
  54. database = {'name': 'foo'}
  55. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  56. def test_database_names_to_dump_passes_through_individual_database_name_with_format():
  57. database = {'name': 'foo', 'format': 'custom'}
  58. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  59. def test_database_names_to_dump_passes_through_all_without_format():
  60. database = {'name': 'all'}
  61. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('all',)
  62. def test_database_names_to_dump_with_all_and_format_and_dry_run_bails():
  63. database = {'name': 'all', 'format': 'custom'}
  64. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  65. 'resolve_credential',
  66. ).replace_with(lambda value, config: value)
  67. flexmock(module).should_receive('execute_command_and_capture_output').never()
  68. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=True) == ()
  69. def test_database_names_to_dump_with_all_and_format_lists_databases():
  70. database = {'name': 'all', 'format': 'custom'}
  71. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  72. 'resolve_credential',
  73. ).replace_with(lambda value, config: value)
  74. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  75. 'foo,test,\nbar,test,"stuff and such"',
  76. )
  77. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  78. 'foo',
  79. 'bar',
  80. )
  81. def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
  82. database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
  83. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  84. 'resolve_credential',
  85. ).replace_with(lambda value, config: value)
  86. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  87. (
  88. 'psql',
  89. '--list',
  90. '--no-password',
  91. '--no-psqlrc',
  92. '--csv',
  93. '--tuples-only',
  94. '--host',
  95. 'localhost',
  96. '--port',
  97. '1234',
  98. ),
  99. environment=object,
  100. ).and_return('foo,test,\nbar,test,"stuff and such"')
  101. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  102. 'foo',
  103. 'bar',
  104. )
  105. def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
  106. database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
  107. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  108. 'resolve_credential',
  109. ).replace_with(lambda value, config: value)
  110. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  111. (
  112. 'psql',
  113. '--list',
  114. '--no-password',
  115. '--no-psqlrc',
  116. '--csv',
  117. '--tuples-only',
  118. '--username',
  119. 'postgres',
  120. ),
  121. environment=object,
  122. ).and_return('foo,test,\nbar,test,"stuff and such"')
  123. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  124. 'foo',
  125. 'bar',
  126. )
  127. def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
  128. database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
  129. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  130. 'resolve_credential',
  131. ).replace_with(lambda value, config: value)
  132. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  133. ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
  134. environment=object,
  135. ).and_return('foo,test,\nbar,test,"stuff and such"')
  136. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  137. 'foo',
  138. 'bar',
  139. )
  140. def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
  141. database = {'name': 'all', 'format': 'custom'}
  142. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  143. 'resolve_credential',
  144. ).replace_with(lambda value, config: value)
  145. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  146. 'foo,test,\ntemplate0,test,blah',
  147. )
  148. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  149. def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
  150. database = {
  151. 'name': 'all',
  152. 'format': 'custom',
  153. 'psql_command': 'docker exec --workdir * mycontainer psql',
  154. }
  155. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  156. 'resolve_credential',
  157. ).replace_with(lambda value, config: value)
  158. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  159. (
  160. 'docker',
  161. 'exec',
  162. '--workdir',
  163. "'*'", # Should get shell escaped to prevent injection attacks.
  164. 'mycontainer',
  165. 'psql',
  166. '--list',
  167. '--no-password',
  168. '--no-psqlrc',
  169. '--csv',
  170. '--tuples-only',
  171. ),
  172. environment=object,
  173. ).and_return('foo,text').once()
  174. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  175. def test_use_streaming_true_for_any_non_directory_format_databases():
  176. assert module.use_streaming(
  177. databases=[{'format': 'stuff'}, {'format': 'directory'}, {}],
  178. config=flexmock(),
  179. )
  180. def test_use_streaming_false_for_all_directory_format_databases():
  181. assert not module.use_streaming(
  182. databases=[{'format': 'directory'}, {'format': 'directory'}],
  183. config=flexmock(),
  184. )
  185. def test_use_streaming_false_for_no_databases():
  186. assert not module.use_streaming(databases=[], config=flexmock())
  187. def test_dump_data_sources_runs_pg_dump_for_each_database():
  188. databases = [{'name': 'foo'}, {'name': 'bar'}]
  189. processes = [flexmock(), flexmock()]
  190. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  191. flexmock(module).should_receive('make_dump_path').and_return('')
  192. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  193. ('bar',),
  194. )
  195. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  196. 'databases/localhost/foo',
  197. ).and_return('databases/localhost/bar')
  198. flexmock(module.os.path).should_receive('exists').and_return(False)
  199. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  200. 'resolve_credential',
  201. ).replace_with(lambda value, config: value)
  202. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  203. for name, process in zip(('foo', 'bar'), processes):
  204. flexmock(module).should_receive('execute_command').with_args(
  205. (
  206. 'pg_dump',
  207. '--no-password',
  208. '--clean',
  209. '--if-exists',
  210. '--format',
  211. 'custom',
  212. name,
  213. '>',
  214. f'databases/localhost/{name}',
  215. ),
  216. shell=True,
  217. environment={'PGSSLMODE': 'disable'},
  218. run_to_completion=False,
  219. ).and_return(process).once()
  220. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  221. '/run/borgmatic',
  222. 'postgresql_databases',
  223. [
  224. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  225. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  226. ],
  227. ).once()
  228. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  229. object,
  230. module.borgmatic.borg.pattern.Pattern(
  231. '/run/borgmatic/postgresql_databases',
  232. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  233. ),
  234. ).once()
  235. assert (
  236. module.dump_data_sources(
  237. databases,
  238. {},
  239. config_paths=('test.yaml',),
  240. borgmatic_runtime_directory='/run/borgmatic',
  241. patterns=[],
  242. dry_run=False,
  243. )
  244. == processes
  245. )
  246. def test_dump_data_sources_raises_when_no_database_names_to_dump():
  247. databases = [{'name': 'foo'}, {'name': 'bar'}]
  248. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  249. flexmock(module).should_receive('make_dump_path').and_return('')
  250. flexmock(module).should_receive('database_names_to_dump').and_return(())
  251. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  252. with pytest.raises(ValueError):
  253. module.dump_data_sources(
  254. databases,
  255. {},
  256. config_paths=('test.yaml',),
  257. borgmatic_runtime_directory='/run/borgmatic',
  258. patterns=[],
  259. dry_run=False,
  260. )
  261. def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
  262. databases = [{'name': 'foo'}, {'name': 'bar'}]
  263. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  264. flexmock(module).should_receive('make_dump_path').and_return('')
  265. flexmock(module).should_receive('database_names_to_dump').and_return(())
  266. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  267. assert (
  268. module.dump_data_sources(
  269. databases,
  270. {},
  271. config_paths=('test.yaml',),
  272. borgmatic_runtime_directory='/run/borgmatic',
  273. patterns=[],
  274. dry_run=True,
  275. )
  276. == []
  277. )
  278. def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
  279. databases = [{'name': 'foo'}, {'name': 'bar'}]
  280. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  281. flexmock(module).should_receive('make_dump_path').and_return('')
  282. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  283. ('bar',),
  284. )
  285. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  286. 'databases/localhost/foo',
  287. ).and_return('databases/localhost/bar')
  288. flexmock(module.os.path).should_receive('exists').and_return(True)
  289. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  290. flexmock(module).should_receive('execute_command').never()
  291. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  292. '/run/borgmatic',
  293. 'postgresql_databases',
  294. [
  295. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  296. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  297. ],
  298. ).once()
  299. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  300. object,
  301. module.borgmatic.borg.pattern.Pattern(
  302. '/run/borgmatic/postgresql_databases',
  303. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  304. ),
  305. ).once()
  306. assert (
  307. module.dump_data_sources(
  308. databases,
  309. {},
  310. config_paths=('test.yaml',),
  311. borgmatic_runtime_directory='/run/borgmatic',
  312. patterns=[],
  313. dry_run=False,
  314. )
  315. == []
  316. )
  317. def test_dump_data_sources_with_dry_run_skips_pg_dump():
  318. databases = [{'name': 'foo'}, {'name': 'bar'}]
  319. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  320. flexmock(module).should_receive('make_dump_path').and_return('')
  321. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  322. ('bar',),
  323. )
  324. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  325. 'databases/localhost/foo',
  326. ).and_return('databases/localhost/bar')
  327. flexmock(module.os.path).should_receive('exists').and_return(False)
  328. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  329. 'resolve_credential',
  330. ).replace_with(lambda value, config: value)
  331. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  332. flexmock(module).should_receive('execute_command').never()
  333. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
  334. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  335. assert (
  336. module.dump_data_sources(
  337. databases,
  338. {},
  339. config_paths=('test.yaml',),
  340. borgmatic_runtime_directory='/run/borgmatic',
  341. patterns=[],
  342. dry_run=True,
  343. )
  344. == []
  345. )
  346. def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
  347. databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
  348. process = flexmock()
  349. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  350. flexmock(module).should_receive('make_dump_path').and_return('')
  351. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  352. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  353. 'databases/database.example.org/foo',
  354. )
  355. flexmock(module.os.path).should_receive('exists').and_return(False)
  356. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  357. 'resolve_credential',
  358. ).replace_with(lambda value, config: value)
  359. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  360. flexmock(module).should_receive('execute_command').with_args(
  361. (
  362. 'pg_dump',
  363. '--no-password',
  364. '--clean',
  365. '--if-exists',
  366. '--host',
  367. 'database.example.org',
  368. '--port',
  369. '5433',
  370. '--format',
  371. 'custom',
  372. 'foo',
  373. '>',
  374. 'databases/database.example.org/foo',
  375. ),
  376. shell=True,
  377. environment={'PGSSLMODE': 'disable'},
  378. run_to_completion=False,
  379. ).and_return(process).once()
  380. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  381. '/run/borgmatic',
  382. 'postgresql_databases',
  383. [
  384. module.borgmatic.actions.restore.Dump(
  385. 'postgresql_databases', 'foo', 'database.example.org', 5433
  386. ),
  387. ],
  388. ).once()
  389. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  390. object,
  391. module.borgmatic.borg.pattern.Pattern(
  392. '/run/borgmatic/postgresql_databases',
  393. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  394. ),
  395. ).once()
  396. assert module.dump_data_sources(
  397. databases,
  398. {},
  399. config_paths=('test.yaml',),
  400. borgmatic_runtime_directory='/run/borgmatic',
  401. patterns=[],
  402. dry_run=False,
  403. ) == [process]
  404. def test_dump_data_sources_runs_pg_dump_with_username_and_password():
  405. databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
  406. process = flexmock()
  407. flexmock(module).should_receive('make_environment').and_return(
  408. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  409. )
  410. flexmock(module).should_receive('make_dump_path').and_return('')
  411. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  412. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  413. 'databases/localhost/foo',
  414. )
  415. flexmock(module.os.path).should_receive('exists').and_return(False)
  416. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  417. 'resolve_credential',
  418. ).replace_with(lambda value, config: value)
  419. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  420. flexmock(module).should_receive('execute_command').with_args(
  421. (
  422. 'pg_dump',
  423. '--no-password',
  424. '--clean',
  425. '--if-exists',
  426. '--username',
  427. 'postgres',
  428. '--format',
  429. 'custom',
  430. 'foo',
  431. '>',
  432. 'databases/localhost/foo',
  433. ),
  434. shell=True,
  435. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  436. run_to_completion=False,
  437. ).and_return(process).once()
  438. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  439. '/run/borgmatic',
  440. 'postgresql_databases',
  441. [
  442. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  443. ],
  444. ).once()
  445. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  446. object,
  447. module.borgmatic.borg.pattern.Pattern(
  448. '/run/borgmatic/postgresql_databases',
  449. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  450. ),
  451. ).once()
  452. assert module.dump_data_sources(
  453. databases,
  454. {},
  455. config_paths=('test.yaml',),
  456. borgmatic_runtime_directory='/run/borgmatic',
  457. patterns=[],
  458. dry_run=False,
  459. ) == [process]
  460. def test_dump_data_sources_with_username_injection_attack_gets_escaped():
  461. databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}]
  462. process = flexmock()
  463. flexmock(module).should_receive('make_environment').and_return(
  464. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  465. )
  466. flexmock(module).should_receive('make_dump_path').and_return('')
  467. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  468. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  469. 'databases/localhost/foo',
  470. )
  471. flexmock(module.os.path).should_receive('exists').and_return(False)
  472. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  473. 'resolve_credential',
  474. ).replace_with(lambda value, config: value)
  475. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  476. flexmock(module).should_receive('execute_command').with_args(
  477. (
  478. 'pg_dump',
  479. '--no-password',
  480. '--clean',
  481. '--if-exists',
  482. '--username',
  483. "'postgres; naughty-command'",
  484. '--format',
  485. 'custom',
  486. 'foo',
  487. '>',
  488. 'databases/localhost/foo',
  489. ),
  490. shell=True,
  491. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  492. run_to_completion=False,
  493. ).and_return(process).once()
  494. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  495. '/run/borgmatic',
  496. 'postgresql_databases',
  497. [
  498. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  499. ],
  500. ).once()
  501. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  502. object,
  503. module.borgmatic.borg.pattern.Pattern(
  504. '/run/borgmatic/postgresql_databases',
  505. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  506. ),
  507. ).once()
  508. assert module.dump_data_sources(
  509. databases,
  510. {},
  511. config_paths=('test.yaml',),
  512. borgmatic_runtime_directory='/run/borgmatic',
  513. patterns=[],
  514. dry_run=False,
  515. ) == [process]
  516. def test_dump_data_sources_runs_pg_dump_with_directory_format():
  517. databases = [{'name': 'foo', 'format': 'directory'}]
  518. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  519. flexmock(module).should_receive('make_dump_path').and_return('')
  520. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  521. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  522. 'databases/localhost/foo',
  523. )
  524. flexmock(module.os.path).should_receive('exists').and_return(False)
  525. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  526. 'resolve_credential',
  527. ).replace_with(lambda value, config: value)
  528. flexmock(module.dump).should_receive('create_parent_directory_for_dump')
  529. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  530. flexmock(module).should_receive('execute_command').with_args(
  531. (
  532. 'pg_dump',
  533. '--no-password',
  534. '--clean',
  535. '--if-exists',
  536. '--format',
  537. 'directory',
  538. '--file',
  539. 'databases/localhost/foo',
  540. 'foo',
  541. ),
  542. shell=True,
  543. environment={'PGSSLMODE': 'disable'},
  544. ).and_return(flexmock()).once()
  545. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  546. '/run/borgmatic',
  547. 'postgresql_databases',
  548. [
  549. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  550. ],
  551. ).once()
  552. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  553. object,
  554. module.borgmatic.borg.pattern.Pattern(
  555. '/run/borgmatic/postgresql_databases',
  556. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  557. ),
  558. ).once()
  559. assert (
  560. module.dump_data_sources(
  561. databases,
  562. {},
  563. config_paths=('test.yaml',),
  564. borgmatic_runtime_directory='/run/borgmatic',
  565. patterns=[],
  566. dry_run=False,
  567. )
  568. == []
  569. )
  570. def test_dump_data_sources_runs_pg_dump_with_string_compression():
  571. databases = [{'name': 'foo', 'compression': 'winrar'}]
  572. processes = [flexmock()]
  573. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  574. flexmock(module).should_receive('make_dump_path').and_return('')
  575. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  576. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  577. 'databases/localhost/foo',
  578. )
  579. flexmock(module.os.path).should_receive('exists').and_return(False)
  580. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  581. 'resolve_credential',
  582. ).replace_with(lambda value, config: value)
  583. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  584. flexmock(module).should_receive('execute_command').with_args(
  585. (
  586. 'pg_dump',
  587. '--no-password',
  588. '--clean',
  589. '--if-exists',
  590. '--format',
  591. 'custom',
  592. '--compress',
  593. 'winrar',
  594. 'foo',
  595. '>',
  596. 'databases/localhost/foo',
  597. ),
  598. shell=True,
  599. environment={'PGSSLMODE': 'disable'},
  600. run_to_completion=False,
  601. ).and_return(processes[0]).once()
  602. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  603. '/run/borgmatic',
  604. 'postgresql_databases',
  605. [
  606. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  607. ],
  608. ).once()
  609. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  610. object,
  611. module.borgmatic.borg.pattern.Pattern(
  612. '/run/borgmatic/postgresql_databases',
  613. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  614. ),
  615. ).once()
  616. assert (
  617. module.dump_data_sources(
  618. databases,
  619. {},
  620. config_paths=('test.yaml',),
  621. borgmatic_runtime_directory='/run/borgmatic',
  622. patterns=[],
  623. dry_run=False,
  624. )
  625. == processes
  626. )
  627. def test_dump_data_sources_runs_pg_dump_with_integer_compression():
  628. databases = [{'name': 'foo', 'compression': 0}]
  629. processes = [flexmock()]
  630. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  631. flexmock(module).should_receive('make_dump_path').and_return('')
  632. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  633. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  634. 'databases/localhost/foo',
  635. )
  636. flexmock(module.os.path).should_receive('exists').and_return(False)
  637. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  638. 'resolve_credential',
  639. ).replace_with(lambda value, config: value)
  640. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  641. flexmock(module).should_receive('execute_command').with_args(
  642. (
  643. 'pg_dump',
  644. '--no-password',
  645. '--clean',
  646. '--if-exists',
  647. '--format',
  648. 'custom',
  649. '--compress',
  650. '0',
  651. 'foo',
  652. '>',
  653. 'databases/localhost/foo',
  654. ),
  655. shell=True,
  656. environment={'PGSSLMODE': 'disable'},
  657. run_to_completion=False,
  658. ).and_return(processes[0]).once()
  659. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  660. '/run/borgmatic',
  661. 'postgresql_databases',
  662. [
  663. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  664. ],
  665. ).once()
  666. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  667. object,
  668. module.borgmatic.borg.pattern.Pattern(
  669. '/run/borgmatic/postgresql_databases',
  670. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  671. ),
  672. ).once()
  673. assert (
  674. module.dump_data_sources(
  675. databases,
  676. {},
  677. config_paths=('test.yaml',),
  678. borgmatic_runtime_directory='/run/borgmatic',
  679. patterns=[],
  680. dry_run=False,
  681. )
  682. == processes
  683. )
  684. def test_dump_data_sources_runs_pg_dump_with_options():
  685. databases = [{'name': 'foo', 'options': '--stuff=such'}]
  686. process = flexmock()
  687. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  688. flexmock(module).should_receive('make_dump_path').and_return('')
  689. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  690. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  691. 'databases/localhost/foo',
  692. )
  693. flexmock(module.os.path).should_receive('exists').and_return(False)
  694. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  695. 'resolve_credential',
  696. ).replace_with(lambda value, config: value)
  697. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  698. flexmock(module).should_receive('execute_command').with_args(
  699. (
  700. 'pg_dump',
  701. '--no-password',
  702. '--clean',
  703. '--if-exists',
  704. '--format',
  705. 'custom',
  706. '--stuff=such',
  707. 'foo',
  708. '>',
  709. 'databases/localhost/foo',
  710. ),
  711. shell=True,
  712. environment={'PGSSLMODE': 'disable'},
  713. run_to_completion=False,
  714. ).and_return(process).once()
  715. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  716. '/run/borgmatic',
  717. 'postgresql_databases',
  718. [
  719. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  720. ],
  721. ).once()
  722. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  723. object,
  724. module.borgmatic.borg.pattern.Pattern(
  725. '/run/borgmatic/postgresql_databases',
  726. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  727. ),
  728. ).once()
  729. assert module.dump_data_sources(
  730. databases,
  731. {},
  732. config_paths=('test.yaml',),
  733. borgmatic_runtime_directory='/run/borgmatic',
  734. patterns=[],
  735. dry_run=False,
  736. ) == [process]
  737. def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
  738. databases = [{'name': 'all'}]
  739. process = flexmock()
  740. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  741. flexmock(module).should_receive('make_dump_path').and_return('')
  742. flexmock(module).should_receive('database_names_to_dump').and_return(('all',))
  743. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  744. 'databases/localhost/all',
  745. )
  746. flexmock(module.os.path).should_receive('exists').and_return(False)
  747. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  748. 'resolve_credential',
  749. ).replace_with(lambda value, config: value)
  750. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  751. flexmock(module).should_receive('execute_command').with_args(
  752. ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
  753. shell=True,
  754. environment={'PGSSLMODE': 'disable'},
  755. run_to_completion=False,
  756. ).and_return(process).once()
  757. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  758. '/run/borgmatic',
  759. 'postgresql_databases',
  760. [
  761. module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'),
  762. ],
  763. ).once()
  764. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  765. object,
  766. module.borgmatic.borg.pattern.Pattern(
  767. '/run/borgmatic/postgresql_databases',
  768. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  769. ),
  770. ).once()
  771. assert module.dump_data_sources(
  772. databases,
  773. {},
  774. config_paths=('test.yaml',),
  775. borgmatic_runtime_directory='/run/borgmatic',
  776. patterns=[],
  777. dry_run=False,
  778. ) == [process]
  779. def test_dump_data_sources_runs_non_default_pg_dump():
  780. databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}]
  781. process = flexmock()
  782. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  783. flexmock(module).should_receive('make_dump_path').and_return('')
  784. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  785. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  786. 'databases/localhost/foo',
  787. )
  788. flexmock(module.os.path).should_receive('exists').and_return(False)
  789. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  790. 'resolve_credential',
  791. ).replace_with(lambda value, config: value)
  792. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  793. flexmock(module).should_receive('execute_command').with_args(
  794. (
  795. 'special_pg_dump',
  796. '--compress',
  797. "'*'", # Should get shell escaped to prevent injection attacks.
  798. '--no-password',
  799. '--clean',
  800. '--if-exists',
  801. '--format',
  802. 'custom',
  803. 'foo',
  804. '>',
  805. 'databases/localhost/foo',
  806. ),
  807. shell=True,
  808. environment={'PGSSLMODE': 'disable'},
  809. run_to_completion=False,
  810. ).and_return(process).once()
  811. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  812. '/run/borgmatic',
  813. 'postgresql_databases',
  814. [
  815. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  816. ],
  817. ).once()
  818. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  819. object,
  820. module.borgmatic.borg.pattern.Pattern(
  821. '/run/borgmatic/postgresql_databases',
  822. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  823. ),
  824. ).once()
  825. assert module.dump_data_sources(
  826. databases,
  827. {},
  828. config_paths=('test.yaml',),
  829. borgmatic_runtime_directory='/run/borgmatic',
  830. patterns=[],
  831. dry_run=False,
  832. ) == [process]
  833. def test_restore_data_source_dump_runs_pg_restore():
  834. hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
  835. extract_process = flexmock(stdout=flexmock())
  836. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  837. 'resolve_credential',
  838. ).replace_with(lambda value, config: value)
  839. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  840. flexmock(module).should_receive('make_dump_path')
  841. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  842. flexmock(module).should_receive('execute_command_with_processes').with_args(
  843. (
  844. 'pg_restore',
  845. '--no-password',
  846. '--if-exists',
  847. '--exit-on-error',
  848. '--clean',
  849. '--dbname',
  850. 'foo',
  851. ),
  852. processes=[extract_process],
  853. output_log_level=logging.DEBUG,
  854. input_file=extract_process.stdout,
  855. environment={'PGSSLMODE': 'disable'},
  856. ).once()
  857. flexmock(module).should_receive('execute_command').with_args(
  858. (
  859. 'psql',
  860. '--no-password',
  861. '--no-psqlrc',
  862. '--quiet',
  863. '--dbname',
  864. 'foo',
  865. '--command',
  866. 'ANALYZE',
  867. ),
  868. environment={'PGSSLMODE': 'disable'},
  869. ).once()
  870. module.restore_data_source_dump(
  871. hook_config,
  872. {},
  873. data_source={'name': 'foo'},
  874. dry_run=False,
  875. extract_process=extract_process,
  876. connection_params={
  877. 'hostname': None,
  878. 'port': None,
  879. 'username': None,
  880. 'password': None,
  881. },
  882. borgmatic_runtime_directory='/run/borgmatic',
  883. )
  884. def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
  885. hook_config = [
  886. {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None},
  887. ]
  888. extract_process = flexmock(stdout=flexmock())
  889. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  890. 'resolve_credential',
  891. ).replace_with(lambda value, config: value)
  892. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  893. flexmock(module).should_receive('make_dump_path')
  894. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  895. flexmock(module).should_receive('execute_command_with_processes').with_args(
  896. (
  897. 'pg_restore',
  898. '--no-password',
  899. '--if-exists',
  900. '--exit-on-error',
  901. '--clean',
  902. '--dbname',
  903. 'foo',
  904. '--host',
  905. 'database.example.org',
  906. '--port',
  907. '5433',
  908. ),
  909. processes=[extract_process],
  910. output_log_level=logging.DEBUG,
  911. input_file=extract_process.stdout,
  912. environment={'PGSSLMODE': 'disable'},
  913. ).once()
  914. flexmock(module).should_receive('execute_command').with_args(
  915. (
  916. 'psql',
  917. '--no-password',
  918. '--no-psqlrc',
  919. '--quiet',
  920. '--host',
  921. 'database.example.org',
  922. '--port',
  923. '5433',
  924. '--dbname',
  925. 'foo',
  926. '--command',
  927. 'ANALYZE',
  928. ),
  929. environment={'PGSSLMODE': 'disable'},
  930. ).once()
  931. module.restore_data_source_dump(
  932. hook_config,
  933. {},
  934. data_source=hook_config[0],
  935. dry_run=False,
  936. extract_process=extract_process,
  937. connection_params={
  938. 'hostname': None,
  939. 'port': None,
  940. 'username': None,
  941. 'password': None,
  942. },
  943. borgmatic_runtime_directory='/run/borgmatic',
  944. )
  945. def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
  946. hook_config = [
  947. {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None},
  948. ]
  949. extract_process = flexmock(stdout=flexmock())
  950. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  951. 'resolve_credential',
  952. ).replace_with(lambda value, config: value)
  953. flexmock(module).should_receive('make_environment').and_return(
  954. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  955. )
  956. flexmock(module).should_receive('make_dump_path')
  957. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  958. flexmock(module).should_receive('execute_command_with_processes').with_args(
  959. (
  960. 'pg_restore',
  961. '--no-password',
  962. '--if-exists',
  963. '--exit-on-error',
  964. '--clean',
  965. '--dbname',
  966. 'foo',
  967. '--username',
  968. 'postgres',
  969. ),
  970. processes=[extract_process],
  971. output_log_level=logging.DEBUG,
  972. input_file=extract_process.stdout,
  973. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  974. ).once()
  975. flexmock(module).should_receive('execute_command').with_args(
  976. (
  977. 'psql',
  978. '--no-password',
  979. '--no-psqlrc',
  980. '--quiet',
  981. '--username',
  982. 'postgres',
  983. '--dbname',
  984. 'foo',
  985. '--command',
  986. 'ANALYZE',
  987. ),
  988. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  989. ).once()
  990. module.restore_data_source_dump(
  991. hook_config,
  992. {},
  993. data_source=hook_config[0],
  994. dry_run=False,
  995. extract_process=extract_process,
  996. connection_params={
  997. 'hostname': None,
  998. 'port': None,
  999. 'username': None,
  1000. 'password': None,
  1001. },
  1002. borgmatic_runtime_directory='/run/borgmatic',
  1003. )
  1004. def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
  1005. hook_config = [
  1006. {
  1007. 'name': 'foo',
  1008. 'hostname': 'database.example.org',
  1009. 'port': 5433,
  1010. 'username': 'postgres',
  1011. 'password': 'trustsome1',
  1012. 'restore_hostname': 'restorehost',
  1013. 'restore_port': 'restoreport',
  1014. 'restore_username': 'restoreusername',
  1015. 'restore_password': 'restorepassword',
  1016. 'schemas': None,
  1017. },
  1018. ]
  1019. extract_process = flexmock(stdout=flexmock())
  1020. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1021. 'resolve_credential',
  1022. ).replace_with(lambda value, config: value)
  1023. flexmock(module).should_receive('make_environment').and_return(
  1024. {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1025. )
  1026. flexmock(module).should_receive('make_dump_path')
  1027. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1028. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1029. (
  1030. 'pg_restore',
  1031. '--no-password',
  1032. '--if-exists',
  1033. '--exit-on-error',
  1034. '--clean',
  1035. '--dbname',
  1036. 'foo',
  1037. '--host',
  1038. 'clihost',
  1039. '--port',
  1040. 'cliport',
  1041. '--username',
  1042. 'cliusername',
  1043. ),
  1044. processes=[extract_process],
  1045. output_log_level=logging.DEBUG,
  1046. input_file=extract_process.stdout,
  1047. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1048. ).once()
  1049. flexmock(module).should_receive('execute_command').with_args(
  1050. (
  1051. 'psql',
  1052. '--no-password',
  1053. '--no-psqlrc',
  1054. '--quiet',
  1055. '--host',
  1056. 'clihost',
  1057. '--port',
  1058. 'cliport',
  1059. '--username',
  1060. 'cliusername',
  1061. '--dbname',
  1062. 'foo',
  1063. '--command',
  1064. 'ANALYZE',
  1065. ),
  1066. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1067. ).once()
  1068. module.restore_data_source_dump(
  1069. hook_config,
  1070. {},
  1071. data_source={'name': 'foo'},
  1072. dry_run=False,
  1073. extract_process=extract_process,
  1074. connection_params={
  1075. 'hostname': 'clihost',
  1076. 'port': 'cliport',
  1077. 'username': 'cliusername',
  1078. 'password': 'clipassword',
  1079. },
  1080. borgmatic_runtime_directory='/run/borgmatic',
  1081. )
  1082. def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore():
  1083. hook_config = [
  1084. {
  1085. 'name': 'foo',
  1086. 'hostname': 'database.example.org',
  1087. 'port': 5433,
  1088. 'username': 'postgres',
  1089. 'password': 'trustsome1',
  1090. 'schemas': None,
  1091. 'restore_hostname': 'restorehost',
  1092. 'restore_port': 'restoreport',
  1093. 'restore_username': 'restoreusername',
  1094. 'restore_password': 'restorepassword',
  1095. },
  1096. ]
  1097. extract_process = flexmock(stdout=flexmock())
  1098. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1099. 'resolve_credential',
  1100. ).replace_with(lambda value, config: value)
  1101. flexmock(module).should_receive('make_environment').and_return(
  1102. {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1103. )
  1104. flexmock(module).should_receive('make_dump_path')
  1105. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1106. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1107. (
  1108. 'pg_restore',
  1109. '--no-password',
  1110. '--if-exists',
  1111. '--exit-on-error',
  1112. '--clean',
  1113. '--dbname',
  1114. 'foo',
  1115. '--host',
  1116. 'restorehost',
  1117. '--port',
  1118. 'restoreport',
  1119. '--username',
  1120. 'restoreusername',
  1121. ),
  1122. processes=[extract_process],
  1123. output_log_level=logging.DEBUG,
  1124. input_file=extract_process.stdout,
  1125. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1126. ).once()
  1127. flexmock(module).should_receive('execute_command').with_args(
  1128. (
  1129. 'psql',
  1130. '--no-password',
  1131. '--no-psqlrc',
  1132. '--quiet',
  1133. '--host',
  1134. 'restorehost',
  1135. '--port',
  1136. 'restoreport',
  1137. '--username',
  1138. 'restoreusername',
  1139. '--dbname',
  1140. 'foo',
  1141. '--command',
  1142. 'ANALYZE',
  1143. ),
  1144. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1145. ).once()
  1146. module.restore_data_source_dump(
  1147. hook_config,
  1148. {},
  1149. data_source=hook_config[0],
  1150. dry_run=False,
  1151. extract_process=extract_process,
  1152. connection_params={
  1153. 'hostname': None,
  1154. 'port': None,
  1155. 'username': None,
  1156. 'password': None,
  1157. },
  1158. borgmatic_runtime_directory='/run/borgmatic',
  1159. )
  1160. def test_restore_data_source_dump_runs_pg_restore_with_options():
  1161. hook_config = [
  1162. {
  1163. 'name': 'foo',
  1164. 'restore_options': '--harder',
  1165. 'analyze_options': '--smarter',
  1166. 'schemas': None,
  1167. },
  1168. ]
  1169. extract_process = flexmock(stdout=flexmock())
  1170. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1171. 'resolve_credential',
  1172. ).replace_with(lambda value, config: value)
  1173. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1174. flexmock(module).should_receive('make_dump_path')
  1175. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1176. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1177. (
  1178. 'pg_restore',
  1179. '--no-password',
  1180. '--if-exists',
  1181. '--exit-on-error',
  1182. '--clean',
  1183. '--dbname',
  1184. 'foo',
  1185. '--harder',
  1186. ),
  1187. processes=[extract_process],
  1188. output_log_level=logging.DEBUG,
  1189. input_file=extract_process.stdout,
  1190. environment={'PGSSLMODE': 'disable'},
  1191. ).once()
  1192. flexmock(module).should_receive('execute_command').with_args(
  1193. (
  1194. 'psql',
  1195. '--no-password',
  1196. '--no-psqlrc',
  1197. '--quiet',
  1198. '--dbname',
  1199. 'foo',
  1200. '--smarter',
  1201. '--command',
  1202. 'ANALYZE',
  1203. ),
  1204. environment={'PGSSLMODE': 'disable'},
  1205. ).once()
  1206. module.restore_data_source_dump(
  1207. hook_config,
  1208. {},
  1209. data_source=hook_config[0],
  1210. dry_run=False,
  1211. extract_process=extract_process,
  1212. connection_params={
  1213. 'hostname': None,
  1214. 'port': None,
  1215. 'username': None,
  1216. 'password': None,
  1217. },
  1218. borgmatic_runtime_directory='/run/borgmatic',
  1219. )
  1220. def test_restore_data_source_dump_runs_psql_for_all_database_dump():
  1221. hook_config = [{'name': 'all', 'schemas': None}]
  1222. extract_process = flexmock(stdout=flexmock())
  1223. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1224. 'resolve_credential',
  1225. ).replace_with(lambda value, config: value)
  1226. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1227. flexmock(module).should_receive('make_dump_path')
  1228. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1229. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1230. (
  1231. 'psql',
  1232. '--no-password',
  1233. '--no-psqlrc',
  1234. ),
  1235. processes=[extract_process],
  1236. output_log_level=logging.DEBUG,
  1237. input_file=extract_process.stdout,
  1238. environment={'PGSSLMODE': 'disable'},
  1239. ).once()
  1240. flexmock(module).should_receive('execute_command').with_args(
  1241. ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
  1242. environment={'PGSSLMODE': 'disable'},
  1243. ).once()
  1244. module.restore_data_source_dump(
  1245. hook_config,
  1246. {},
  1247. data_source={'name': 'all'},
  1248. dry_run=False,
  1249. extract_process=extract_process,
  1250. connection_params={
  1251. 'hostname': None,
  1252. 'port': None,
  1253. 'username': None,
  1254. 'password': None,
  1255. },
  1256. borgmatic_runtime_directory='/run/borgmatic',
  1257. )
  1258. def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
  1259. hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
  1260. extract_process = flexmock(stdout=flexmock())
  1261. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1262. 'resolve_credential',
  1263. ).replace_with(lambda value, config: value)
  1264. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1265. flexmock(module).should_receive('make_dump_path')
  1266. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1267. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1268. ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
  1269. processes=[extract_process],
  1270. output_log_level=logging.DEBUG,
  1271. input_file=extract_process.stdout,
  1272. environment={'PGSSLMODE': 'disable'},
  1273. ).once()
  1274. flexmock(module).should_receive('execute_command').with_args(
  1275. (
  1276. 'psql',
  1277. '--no-password',
  1278. '--no-psqlrc',
  1279. '--quiet',
  1280. '--dbname',
  1281. 'foo',
  1282. '--command',
  1283. 'ANALYZE',
  1284. ),
  1285. environment={'PGSSLMODE': 'disable'},
  1286. ).once()
  1287. module.restore_data_source_dump(
  1288. hook_config,
  1289. {},
  1290. data_source=hook_config[0],
  1291. dry_run=False,
  1292. extract_process=extract_process,
  1293. connection_params={
  1294. 'hostname': None,
  1295. 'port': None,
  1296. 'username': None,
  1297. 'password': None,
  1298. },
  1299. borgmatic_runtime_directory='/run/borgmatic',
  1300. )
  1301. def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
  1302. hook_config = [
  1303. {
  1304. 'name': 'foo',
  1305. 'pg_restore_command': 'docker exec --workdir * mycontainer pg_restore',
  1306. 'psql_command': 'docker exec --workdir * mycontainer psql',
  1307. 'schemas': None,
  1308. },
  1309. ]
  1310. extract_process = flexmock(stdout=flexmock())
  1311. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1312. 'resolve_credential',
  1313. ).replace_with(lambda value, config: value)
  1314. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1315. flexmock(module).should_receive('make_dump_path')
  1316. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1317. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1318. (
  1319. 'docker',
  1320. 'exec',
  1321. '--workdir',
  1322. "'*'", # Should get shell escaped to prevent injection attacks.
  1323. 'mycontainer',
  1324. 'pg_restore',
  1325. '--no-password',
  1326. '--if-exists',
  1327. '--exit-on-error',
  1328. '--clean',
  1329. '--dbname',
  1330. 'foo',
  1331. ),
  1332. processes=[extract_process],
  1333. output_log_level=logging.DEBUG,
  1334. input_file=extract_process.stdout,
  1335. environment={'PGSSLMODE': 'disable'},
  1336. ).once()
  1337. flexmock(module).should_receive('execute_command').with_args(
  1338. (
  1339. 'docker',
  1340. 'exec',
  1341. '--workdir',
  1342. "'*'", # Should get shell escaped to prevent injection attacks.
  1343. 'mycontainer',
  1344. 'psql',
  1345. '--no-password',
  1346. '--no-psqlrc',
  1347. '--quiet',
  1348. '--dbname',
  1349. 'foo',
  1350. '--command',
  1351. 'ANALYZE',
  1352. ),
  1353. environment={'PGSSLMODE': 'disable'},
  1354. ).once()
  1355. module.restore_data_source_dump(
  1356. hook_config,
  1357. {},
  1358. data_source=hook_config[0],
  1359. dry_run=False,
  1360. extract_process=extract_process,
  1361. connection_params={
  1362. 'hostname': None,
  1363. 'port': None,
  1364. 'username': None,
  1365. 'password': None,
  1366. },
  1367. borgmatic_runtime_directory='/run/borgmatic',
  1368. )
  1369. def test_restore_data_source_dump_with_dry_run_skips_restore():
  1370. hook_config = [{'name': 'foo', 'schemas': None}]
  1371. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1372. 'resolve_credential',
  1373. ).replace_with(lambda value, config: value)
  1374. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1375. flexmock(module).should_receive('make_dump_path')
  1376. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1377. flexmock(module).should_receive('execute_command_with_processes').never()
  1378. module.restore_data_source_dump(
  1379. hook_config,
  1380. {},
  1381. data_source={'name': 'foo'},
  1382. dry_run=True,
  1383. extract_process=flexmock(),
  1384. connection_params={
  1385. 'hostname': None,
  1386. 'port': None,
  1387. 'username': None,
  1388. 'password': None,
  1389. },
  1390. borgmatic_runtime_directory='/run/borgmatic',
  1391. )
  1392. def test_restore_data_source_dump_without_extract_process_restores_from_disk():
  1393. hook_config = [{'name': 'foo', 'schemas': None}]
  1394. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1395. 'resolve_credential',
  1396. ).replace_with(lambda value, config: value)
  1397. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1398. flexmock(module).should_receive('make_dump_path')
  1399. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1400. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1401. (
  1402. 'pg_restore',
  1403. '--no-password',
  1404. '--if-exists',
  1405. '--exit-on-error',
  1406. '--clean',
  1407. '--dbname',
  1408. 'foo',
  1409. '/dump/path',
  1410. ),
  1411. processes=[],
  1412. output_log_level=logging.DEBUG,
  1413. input_file=None,
  1414. environment={'PGSSLMODE': 'disable'},
  1415. ).once()
  1416. flexmock(module).should_receive('execute_command').with_args(
  1417. (
  1418. 'psql',
  1419. '--no-password',
  1420. '--no-psqlrc',
  1421. '--quiet',
  1422. '--dbname',
  1423. 'foo',
  1424. '--command',
  1425. 'ANALYZE',
  1426. ),
  1427. environment={'PGSSLMODE': 'disable'},
  1428. ).once()
  1429. module.restore_data_source_dump(
  1430. hook_config,
  1431. {},
  1432. data_source={'name': 'foo'},
  1433. dry_run=False,
  1434. extract_process=None,
  1435. connection_params={
  1436. 'hostname': None,
  1437. 'port': None,
  1438. 'username': None,
  1439. 'password': None,
  1440. },
  1441. borgmatic_runtime_directory='/run/borgmatic',
  1442. )
  1443. def test_restore_data_source_dump_with_schemas_restores_schemas():
  1444. hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
  1445. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1446. 'resolve_credential',
  1447. ).replace_with(lambda value, config: value)
  1448. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1449. flexmock(module).should_receive('make_dump_path')
  1450. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1451. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1452. (
  1453. 'pg_restore',
  1454. '--no-password',
  1455. '--if-exists',
  1456. '--exit-on-error',
  1457. '--clean',
  1458. '--dbname',
  1459. 'foo',
  1460. '/dump/path',
  1461. '--schema',
  1462. 'bar',
  1463. '--schema',
  1464. 'baz',
  1465. ),
  1466. processes=[],
  1467. output_log_level=logging.DEBUG,
  1468. input_file=None,
  1469. environment={'PGSSLMODE': 'disable'},
  1470. ).once()
  1471. flexmock(module).should_receive('execute_command').with_args(
  1472. (
  1473. 'psql',
  1474. '--no-password',
  1475. '--no-psqlrc',
  1476. '--quiet',
  1477. '--dbname',
  1478. 'foo',
  1479. '--command',
  1480. 'ANALYZE',
  1481. ),
  1482. environment={'PGSSLMODE': 'disable'},
  1483. ).once()
  1484. module.restore_data_source_dump(
  1485. hook_config,
  1486. {},
  1487. data_source=hook_config[0],
  1488. dry_run=False,
  1489. extract_process=None,
  1490. connection_params={
  1491. 'hostname': None,
  1492. 'port': None,
  1493. 'username': None,
  1494. 'password': None,
  1495. },
  1496. borgmatic_runtime_directory='/run/borgmatic',
  1497. )