test_postgresql.py 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686
  1. import logging
  2. import pytest
  3. from flexmock import flexmock
  4. from borgmatic.hooks.data_source import postgresql as module
  5. def test_make_environment_maps_options_to_environment():
  6. database = {
  7. 'name': 'foo',
  8. 'password': 'pass',
  9. 'ssl_mode': 'require',
  10. 'ssl_cert': 'cert.crt',
  11. 'ssl_key': 'key.key',
  12. 'ssl_root_cert': 'root.crt',
  13. 'ssl_crl': 'crl.crl',
  14. }
  15. expected = {
  16. 'USER': 'root',
  17. 'PGPASSWORD': 'pass',
  18. 'PGSSLMODE': 'require',
  19. 'PGSSLCERT': 'cert.crt',
  20. 'PGSSLKEY': 'key.key',
  21. 'PGSSLROOTCERT': 'root.crt',
  22. 'PGSSLCRL': 'crl.crl',
  23. }
  24. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  25. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  26. 'resolve_credential',
  27. ).replace_with(lambda value, config: value)
  28. assert module.make_environment(database, {}) == expected
  29. def test_make_environment_with_cli_password_sets_correct_password():
  30. database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
  31. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  32. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  33. 'resolve_credential',
  34. ).replace_with(lambda value, config: value)
  35. environment = module.make_environment(
  36. database,
  37. {},
  38. restore_connection_params={'password': 'clipassword'},
  39. )
  40. assert environment['PGPASSWORD'] == 'clipassword'
  41. def test_make_environment_without_cli_password_or_configured_password_does_not_set_password():
  42. database = {'name': 'foo'}
  43. environment = module.make_environment(
  44. database,
  45. {},
  46. restore_connection_params={'username': 'someone'},
  47. )
  48. assert 'PGPASSWORD' not in environment
  49. def test_make_environment_without_ssl_mode_does_not_set_ssl_mode():
  50. database = {'name': 'foo'}
  51. environment = module.make_environment(database, {})
  52. assert 'PGSSLMODE' not in environment
  53. def test_database_names_to_dump_passes_through_individual_database_name():
  54. database = {'name': 'foo'}
  55. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  56. def test_database_names_to_dump_passes_through_individual_database_name_with_format():
  57. database = {'name': 'foo', 'format': 'custom'}
  58. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  59. def test_database_names_to_dump_passes_through_all_without_format():
  60. database = {'name': 'all'}
  61. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('all',)
  62. def test_database_names_to_dump_with_all_and_format_and_dry_run_bails():
  63. database = {'name': 'all', 'format': 'custom'}
  64. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  65. 'resolve_credential',
  66. ).replace_with(lambda value, config: value)
  67. flexmock(module).should_receive('execute_command_and_capture_output').never()
  68. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=True) == ()
  69. def test_database_names_to_dump_with_all_and_format_lists_databases():
  70. database = {'name': 'all', 'format': 'custom'}
  71. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  72. 'resolve_credential',
  73. ).replace_with(lambda value, config: value)
  74. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  75. 'foo,test,\nbar,test,"stuff and such"',
  76. )
  77. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  78. 'foo',
  79. 'bar',
  80. )
  81. def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
  82. database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
  83. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  84. 'resolve_credential',
  85. ).replace_with(lambda value, config: value)
  86. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  87. (
  88. 'psql',
  89. '--list',
  90. '--no-password',
  91. '--no-psqlrc',
  92. '--csv',
  93. '--tuples-only',
  94. '--host',
  95. 'localhost',
  96. '--port',
  97. '1234',
  98. ),
  99. environment=object,
  100. working_directory=None,
  101. ).and_return('foo,test,\nbar,test,"stuff and such"')
  102. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  103. 'foo',
  104. 'bar',
  105. )
  106. def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
  107. database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
  108. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  109. 'resolve_credential',
  110. ).replace_with(lambda value, config: value)
  111. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  112. (
  113. 'psql',
  114. '--list',
  115. '--no-password',
  116. '--no-psqlrc',
  117. '--csv',
  118. '--tuples-only',
  119. '--username',
  120. 'postgres',
  121. ),
  122. environment=object,
  123. working_directory=None,
  124. ).and_return('foo,test,\nbar,test,"stuff and such"')
  125. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  126. 'foo',
  127. 'bar',
  128. )
  129. def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
  130. database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
  131. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  132. 'resolve_credential',
  133. ).replace_with(lambda value, config: value)
  134. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  135. ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
  136. environment=object,
  137. working_directory=None,
  138. ).and_return('foo,test,\nbar,test,"stuff and such"')
  139. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  140. 'foo',
  141. 'bar',
  142. )
  143. def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
  144. database = {'name': 'all', 'format': 'custom'}
  145. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  146. 'resolve_credential',
  147. ).replace_with(lambda value, config: value)
  148. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  149. 'foo,test,\ntemplate0,test,blah',
  150. )
  151. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  152. def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
  153. database = {
  154. 'name': 'all',
  155. 'format': 'custom',
  156. 'psql_command': 'docker exec --workdir * mycontainer psql',
  157. }
  158. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  159. 'resolve_credential',
  160. ).replace_with(lambda value, config: value)
  161. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  162. (
  163. 'docker',
  164. 'exec',
  165. '--workdir',
  166. "'*'", # Should get shell escaped to prevent injection attacks.
  167. 'mycontainer',
  168. 'psql',
  169. '--list',
  170. '--no-password',
  171. '--no-psqlrc',
  172. '--csv',
  173. '--tuples-only',
  174. ),
  175. environment=object,
  176. working_directory=None,
  177. ).and_return('foo,text').once()
  178. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  179. def test_use_streaming_true_for_any_non_directory_format_databases():
  180. assert module.use_streaming(
  181. databases=[{'format': 'stuff'}, {'format': 'directory'}, {}],
  182. config=flexmock(),
  183. )
  184. def test_use_streaming_false_for_all_directory_format_databases():
  185. assert not module.use_streaming(
  186. databases=[{'format': 'directory'}, {'format': 'directory'}],
  187. config=flexmock(),
  188. )
  189. def test_use_streaming_false_for_no_databases():
  190. assert not module.use_streaming(databases=[], config=flexmock())
  191. def test_dump_data_sources_runs_pg_dump_for_each_database():
  192. databases = [{'name': 'foo'}, {'name': 'bar'}]
  193. processes = [flexmock(), flexmock()]
  194. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  195. flexmock(module).should_receive('make_dump_path').and_return('')
  196. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  197. ('bar',),
  198. )
  199. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  200. 'databases/localhost/foo',
  201. ).and_return('databases/localhost/bar')
  202. flexmock(module.os.path).should_receive('exists').and_return(False)
  203. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  204. 'resolve_credential',
  205. ).replace_with(lambda value, config: value)
  206. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  207. for name, process in zip(('foo', 'bar'), processes):
  208. flexmock(module).should_receive('execute_command').with_args(
  209. (
  210. 'pg_dump',
  211. '--no-password',
  212. '--clean',
  213. '--if-exists',
  214. '--format',
  215. 'custom',
  216. name,
  217. '>',
  218. f'databases/localhost/{name}',
  219. ),
  220. shell=True,
  221. environment={'PGSSLMODE': 'disable'},
  222. run_to_completion=False,
  223. working_directory=None,
  224. ).and_return(process).once()
  225. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  226. '/run/borgmatic',
  227. 'postgresql_databases',
  228. [
  229. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  230. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  231. ],
  232. ).once()
  233. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  234. object,
  235. module.borgmatic.borg.pattern.Pattern(
  236. '/run/borgmatic/postgresql_databases',
  237. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  238. ),
  239. ).once()
  240. assert (
  241. module.dump_data_sources(
  242. databases,
  243. {},
  244. config_paths=('test.yaml',),
  245. borgmatic_runtime_directory='/run/borgmatic',
  246. patterns=[],
  247. dry_run=False,
  248. )
  249. == processes
  250. )
  251. def test_dump_data_sources_raises_when_no_database_names_to_dump():
  252. databases = [{'name': 'foo'}, {'name': 'bar'}]
  253. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  254. flexmock(module).should_receive('make_dump_path').and_return('')
  255. flexmock(module).should_receive('database_names_to_dump').and_return(())
  256. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  257. with pytest.raises(ValueError):
  258. module.dump_data_sources(
  259. databases,
  260. {},
  261. config_paths=('test.yaml',),
  262. borgmatic_runtime_directory='/run/borgmatic',
  263. patterns=[],
  264. dry_run=False,
  265. )
  266. def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
  267. databases = [{'name': 'foo'}, {'name': 'bar'}]
  268. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  269. flexmock(module).should_receive('make_dump_path').and_return('')
  270. flexmock(module).should_receive('database_names_to_dump').and_return(())
  271. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  272. assert (
  273. module.dump_data_sources(
  274. databases,
  275. {},
  276. config_paths=('test.yaml',),
  277. borgmatic_runtime_directory='/run/borgmatic',
  278. patterns=[],
  279. dry_run=True,
  280. )
  281. == []
  282. )
  283. def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
  284. databases = [{'name': 'foo'}, {'name': 'bar'}]
  285. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  286. flexmock(module).should_receive('make_dump_path').and_return('')
  287. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  288. ('bar',),
  289. )
  290. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  291. 'databases/localhost/foo',
  292. ).and_return('databases/localhost/bar')
  293. flexmock(module.os.path).should_receive('exists').and_return(True)
  294. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  295. flexmock(module).should_receive('execute_command').never()
  296. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  297. '/run/borgmatic',
  298. 'postgresql_databases',
  299. [
  300. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  301. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  302. ],
  303. ).once()
  304. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  305. object,
  306. module.borgmatic.borg.pattern.Pattern(
  307. '/run/borgmatic/postgresql_databases',
  308. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  309. ),
  310. ).once()
  311. assert (
  312. module.dump_data_sources(
  313. databases,
  314. {},
  315. config_paths=('test.yaml',),
  316. borgmatic_runtime_directory='/run/borgmatic',
  317. patterns=[],
  318. dry_run=False,
  319. )
  320. == []
  321. )
  322. def test_dump_data_sources_with_dry_run_skips_pg_dump():
  323. databases = [{'name': 'foo'}, {'name': 'bar'}]
  324. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  325. flexmock(module).should_receive('make_dump_path').and_return('')
  326. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  327. ('bar',),
  328. )
  329. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  330. 'databases/localhost/foo',
  331. ).and_return('databases/localhost/bar')
  332. flexmock(module.os.path).should_receive('exists').and_return(False)
  333. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  334. 'resolve_credential',
  335. ).replace_with(lambda value, config: value)
  336. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  337. flexmock(module).should_receive('execute_command').never()
  338. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
  339. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  340. assert (
  341. module.dump_data_sources(
  342. databases,
  343. {},
  344. config_paths=('test.yaml',),
  345. borgmatic_runtime_directory='/run/borgmatic',
  346. patterns=[],
  347. dry_run=True,
  348. )
  349. == []
  350. )
  351. def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
  352. databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
  353. process = flexmock()
  354. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  355. flexmock(module).should_receive('make_dump_path').and_return('')
  356. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  357. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  358. 'databases/database.example.org/foo',
  359. )
  360. flexmock(module.os.path).should_receive('exists').and_return(False)
  361. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  362. 'resolve_credential',
  363. ).replace_with(lambda value, config: value)
  364. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  365. flexmock(module).should_receive('execute_command').with_args(
  366. (
  367. 'pg_dump',
  368. '--no-password',
  369. '--clean',
  370. '--if-exists',
  371. '--host',
  372. 'database.example.org',
  373. '--port',
  374. '5433',
  375. '--format',
  376. 'custom',
  377. 'foo',
  378. '>',
  379. 'databases/database.example.org/foo',
  380. ),
  381. shell=True,
  382. environment={'PGSSLMODE': 'disable'},
  383. run_to_completion=False,
  384. working_directory=None,
  385. ).and_return(process).once()
  386. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  387. '/run/borgmatic',
  388. 'postgresql_databases',
  389. [
  390. module.borgmatic.actions.restore.Dump(
  391. 'postgresql_databases', 'foo', 'database.example.org', 5433
  392. ),
  393. ],
  394. ).once()
  395. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  396. object,
  397. module.borgmatic.borg.pattern.Pattern(
  398. '/run/borgmatic/postgresql_databases',
  399. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  400. ),
  401. ).once()
  402. assert module.dump_data_sources(
  403. databases,
  404. {},
  405. config_paths=('test.yaml',),
  406. borgmatic_runtime_directory='/run/borgmatic',
  407. patterns=[],
  408. dry_run=False,
  409. ) == [process]
  410. def test_dump_data_sources_runs_pg_dump_with_username_and_password():
  411. databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
  412. process = flexmock()
  413. flexmock(module).should_receive('make_environment').and_return(
  414. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  415. )
  416. flexmock(module).should_receive('make_dump_path').and_return('')
  417. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  418. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  419. 'databases/localhost/foo',
  420. )
  421. flexmock(module.os.path).should_receive('exists').and_return(False)
  422. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  423. 'resolve_credential',
  424. ).replace_with(lambda value, config: value)
  425. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  426. flexmock(module).should_receive('execute_command').with_args(
  427. (
  428. 'pg_dump',
  429. '--no-password',
  430. '--clean',
  431. '--if-exists',
  432. '--username',
  433. 'postgres',
  434. '--format',
  435. 'custom',
  436. 'foo',
  437. '>',
  438. 'databases/localhost/foo',
  439. ),
  440. shell=True,
  441. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  442. run_to_completion=False,
  443. working_directory=None,
  444. ).and_return(process).once()
  445. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  446. '/run/borgmatic',
  447. 'postgresql_databases',
  448. [
  449. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  450. ],
  451. ).once()
  452. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  453. object,
  454. module.borgmatic.borg.pattern.Pattern(
  455. '/run/borgmatic/postgresql_databases',
  456. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  457. ),
  458. ).once()
  459. assert module.dump_data_sources(
  460. databases,
  461. {},
  462. config_paths=('test.yaml',),
  463. borgmatic_runtime_directory='/run/borgmatic',
  464. patterns=[],
  465. dry_run=False,
  466. ) == [process]
  467. def test_dump_data_sources_with_username_injection_attack_gets_escaped():
  468. databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}]
  469. process = flexmock()
  470. flexmock(module).should_receive('make_environment').and_return(
  471. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  472. )
  473. flexmock(module).should_receive('make_dump_path').and_return('')
  474. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  475. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  476. 'databases/localhost/foo',
  477. )
  478. flexmock(module.os.path).should_receive('exists').and_return(False)
  479. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  480. 'resolve_credential',
  481. ).replace_with(lambda value, config: value)
  482. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  483. flexmock(module).should_receive('execute_command').with_args(
  484. (
  485. 'pg_dump',
  486. '--no-password',
  487. '--clean',
  488. '--if-exists',
  489. '--username',
  490. "'postgres; naughty-command'",
  491. '--format',
  492. 'custom',
  493. 'foo',
  494. '>',
  495. 'databases/localhost/foo',
  496. ),
  497. shell=True,
  498. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  499. run_to_completion=False,
  500. working_directory=None,
  501. ).and_return(process).once()
  502. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  503. '/run/borgmatic',
  504. 'postgresql_databases',
  505. [
  506. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  507. ],
  508. ).once()
  509. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  510. object,
  511. module.borgmatic.borg.pattern.Pattern(
  512. '/run/borgmatic/postgresql_databases',
  513. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  514. ),
  515. ).once()
  516. assert module.dump_data_sources(
  517. databases,
  518. {},
  519. config_paths=('test.yaml',),
  520. borgmatic_runtime_directory='/run/borgmatic',
  521. patterns=[],
  522. dry_run=False,
  523. ) == [process]
  524. def test_dump_data_sources_runs_pg_dump_with_directory_format():
  525. databases = [{'name': 'foo', 'format': 'directory'}]
  526. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  527. flexmock(module).should_receive('make_dump_path').and_return('')
  528. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  529. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  530. 'databases/localhost/foo',
  531. )
  532. flexmock(module.os.path).should_receive('exists').and_return(False)
  533. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  534. 'resolve_credential',
  535. ).replace_with(lambda value, config: value)
  536. flexmock(module.dump).should_receive('create_parent_directory_for_dump')
  537. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  538. flexmock(module).should_receive('execute_command').with_args(
  539. (
  540. 'pg_dump',
  541. '--no-password',
  542. '--clean',
  543. '--if-exists',
  544. '--format',
  545. 'directory',
  546. '--file',
  547. 'databases/localhost/foo',
  548. 'foo',
  549. ),
  550. shell=True,
  551. environment={'PGSSLMODE': 'disable'},
  552. working_directory=None,
  553. ).and_return(flexmock()).once()
  554. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  555. '/run/borgmatic',
  556. 'postgresql_databases',
  557. [
  558. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  559. ],
  560. ).once()
  561. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  562. object,
  563. module.borgmatic.borg.pattern.Pattern(
  564. '/run/borgmatic/postgresql_databases',
  565. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  566. ),
  567. ).once()
  568. assert (
  569. module.dump_data_sources(
  570. databases,
  571. {},
  572. config_paths=('test.yaml',),
  573. borgmatic_runtime_directory='/run/borgmatic',
  574. patterns=[],
  575. dry_run=False,
  576. )
  577. == []
  578. )
  579. def test_dump_data_sources_runs_pg_dump_with_string_compression():
  580. databases = [{'name': 'foo', 'compression': 'winrar'}]
  581. processes = [flexmock()]
  582. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  583. flexmock(module).should_receive('make_dump_path').and_return('')
  584. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  585. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  586. 'databases/localhost/foo',
  587. )
  588. flexmock(module.os.path).should_receive('exists').and_return(False)
  589. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  590. 'resolve_credential',
  591. ).replace_with(lambda value, config: value)
  592. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  593. flexmock(module).should_receive('execute_command').with_args(
  594. (
  595. 'pg_dump',
  596. '--no-password',
  597. '--clean',
  598. '--if-exists',
  599. '--format',
  600. 'custom',
  601. '--compress',
  602. 'winrar',
  603. 'foo',
  604. '>',
  605. 'databases/localhost/foo',
  606. ),
  607. shell=True,
  608. environment={'PGSSLMODE': 'disable'},
  609. run_to_completion=False,
  610. working_directory=None,
  611. ).and_return(processes[0]).once()
  612. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  613. '/run/borgmatic',
  614. 'postgresql_databases',
  615. [
  616. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  617. ],
  618. ).once()
  619. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  620. object,
  621. module.borgmatic.borg.pattern.Pattern(
  622. '/run/borgmatic/postgresql_databases',
  623. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  624. ),
  625. ).once()
  626. assert (
  627. module.dump_data_sources(
  628. databases,
  629. {},
  630. config_paths=('test.yaml',),
  631. borgmatic_runtime_directory='/run/borgmatic',
  632. patterns=[],
  633. dry_run=False,
  634. )
  635. == processes
  636. )
  637. def test_dump_data_sources_runs_pg_dump_with_integer_compression():
  638. databases = [{'name': 'foo', 'compression': 0}]
  639. processes = [flexmock()]
  640. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  641. flexmock(module).should_receive('make_dump_path').and_return('')
  642. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  643. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  644. 'databases/localhost/foo',
  645. )
  646. flexmock(module.os.path).should_receive('exists').and_return(False)
  647. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  648. 'resolve_credential',
  649. ).replace_with(lambda value, config: value)
  650. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  651. flexmock(module).should_receive('execute_command').with_args(
  652. (
  653. 'pg_dump',
  654. '--no-password',
  655. '--clean',
  656. '--if-exists',
  657. '--format',
  658. 'custom',
  659. '--compress',
  660. '0',
  661. 'foo',
  662. '>',
  663. 'databases/localhost/foo',
  664. ),
  665. shell=True,
  666. environment={'PGSSLMODE': 'disable'},
  667. run_to_completion=False,
  668. working_directory=None,
  669. ).and_return(processes[0]).once()
  670. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  671. '/run/borgmatic',
  672. 'postgresql_databases',
  673. [
  674. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  675. ],
  676. ).once()
  677. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  678. object,
  679. module.borgmatic.borg.pattern.Pattern(
  680. '/run/borgmatic/postgresql_databases',
  681. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  682. ),
  683. ).once()
  684. assert (
  685. module.dump_data_sources(
  686. databases,
  687. {},
  688. config_paths=('test.yaml',),
  689. borgmatic_runtime_directory='/run/borgmatic',
  690. patterns=[],
  691. dry_run=False,
  692. )
  693. == processes
  694. )
  695. def test_dump_data_sources_runs_pg_dump_with_options():
  696. databases = [{'name': 'foo', 'options': '--stuff=such'}]
  697. process = flexmock()
  698. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  699. flexmock(module).should_receive('make_dump_path').and_return('')
  700. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  701. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  702. 'databases/localhost/foo',
  703. )
  704. flexmock(module.os.path).should_receive('exists').and_return(False)
  705. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  706. 'resolve_credential',
  707. ).replace_with(lambda value, config: value)
  708. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  709. flexmock(module).should_receive('execute_command').with_args(
  710. (
  711. 'pg_dump',
  712. '--no-password',
  713. '--clean',
  714. '--if-exists',
  715. '--format',
  716. 'custom',
  717. '--stuff=such',
  718. 'foo',
  719. '>',
  720. 'databases/localhost/foo',
  721. ),
  722. shell=True,
  723. environment={'PGSSLMODE': 'disable'},
  724. run_to_completion=False,
  725. working_directory=None,
  726. ).and_return(process).once()
  727. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  728. '/run/borgmatic',
  729. 'postgresql_databases',
  730. [
  731. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  732. ],
  733. ).once()
  734. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  735. object,
  736. module.borgmatic.borg.pattern.Pattern(
  737. '/run/borgmatic/postgresql_databases',
  738. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  739. ),
  740. ).once()
  741. assert module.dump_data_sources(
  742. databases,
  743. {},
  744. config_paths=('test.yaml',),
  745. borgmatic_runtime_directory='/run/borgmatic',
  746. patterns=[],
  747. dry_run=False,
  748. ) == [process]
  749. def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
  750. databases = [{'name': 'all'}]
  751. process = flexmock()
  752. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  753. flexmock(module).should_receive('make_dump_path').and_return('')
  754. flexmock(module).should_receive('database_names_to_dump').and_return(('all',))
  755. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  756. 'databases/localhost/all',
  757. )
  758. flexmock(module.os.path).should_receive('exists').and_return(False)
  759. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  760. 'resolve_credential',
  761. ).replace_with(lambda value, config: value)
  762. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  763. flexmock(module).should_receive('execute_command').with_args(
  764. ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
  765. shell=True,
  766. environment={'PGSSLMODE': 'disable'},
  767. run_to_completion=False,
  768. working_directory=None,
  769. ).and_return(process).once()
  770. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  771. '/run/borgmatic',
  772. 'postgresql_databases',
  773. [
  774. module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'),
  775. ],
  776. ).once()
  777. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  778. object,
  779. module.borgmatic.borg.pattern.Pattern(
  780. '/run/borgmatic/postgresql_databases',
  781. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  782. ),
  783. ).once()
  784. assert module.dump_data_sources(
  785. databases,
  786. {},
  787. config_paths=('test.yaml',),
  788. borgmatic_runtime_directory='/run/borgmatic',
  789. patterns=[],
  790. dry_run=False,
  791. ) == [process]
  792. def test_dump_data_sources_runs_non_default_pg_dump():
  793. databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}]
  794. process = flexmock()
  795. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  796. flexmock(module).should_receive('make_dump_path').and_return('')
  797. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  798. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  799. 'databases/localhost/foo',
  800. )
  801. flexmock(module.os.path).should_receive('exists').and_return(False)
  802. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  803. 'resolve_credential',
  804. ).replace_with(lambda value, config: value)
  805. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  806. flexmock(module).should_receive('execute_command').with_args(
  807. (
  808. 'special_pg_dump',
  809. '--compress',
  810. "'*'", # Should get shell escaped to prevent injection attacks.
  811. '--no-password',
  812. '--clean',
  813. '--if-exists',
  814. '--format',
  815. 'custom',
  816. 'foo',
  817. '>',
  818. 'databases/localhost/foo',
  819. ),
  820. shell=True,
  821. environment={'PGSSLMODE': 'disable'},
  822. run_to_completion=False,
  823. working_directory=None,
  824. ).and_return(process).once()
  825. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  826. '/run/borgmatic',
  827. 'postgresql_databases',
  828. [
  829. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  830. ],
  831. ).once()
  832. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  833. object,
  834. module.borgmatic.borg.pattern.Pattern(
  835. '/run/borgmatic/postgresql_databases',
  836. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  837. ),
  838. ).once()
  839. assert module.dump_data_sources(
  840. databases,
  841. {},
  842. config_paths=('test.yaml',),
  843. borgmatic_runtime_directory='/run/borgmatic',
  844. patterns=[],
  845. dry_run=False,
  846. ) == [process]
  847. def test_restore_data_source_dump_runs_pg_restore():
  848. hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
  849. extract_process = flexmock(stdout=flexmock())
  850. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  851. 'resolve_credential',
  852. ).replace_with(lambda value, config: value)
  853. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  854. flexmock(module).should_receive('make_dump_path')
  855. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  856. flexmock(module).should_receive('execute_command_with_processes').with_args(
  857. (
  858. 'pg_restore',
  859. '--no-password',
  860. '--if-exists',
  861. '--exit-on-error',
  862. '--clean',
  863. '--dbname',
  864. 'foo',
  865. ),
  866. processes=[extract_process],
  867. output_log_level=logging.DEBUG,
  868. input_file=extract_process.stdout,
  869. environment={'PGSSLMODE': 'disable'},
  870. working_directory=None,
  871. ).once()
  872. flexmock(module).should_receive('execute_command').with_args(
  873. (
  874. 'psql',
  875. '--no-password',
  876. '--no-psqlrc',
  877. '--quiet',
  878. '--dbname',
  879. 'foo',
  880. '--command',
  881. 'ANALYZE',
  882. ),
  883. environment={'PGSSLMODE': 'disable'},
  884. working_directory=None,
  885. ).once()
  886. module.restore_data_source_dump(
  887. hook_config,
  888. {},
  889. data_source={'name': 'foo'},
  890. dry_run=False,
  891. extract_process=extract_process,
  892. connection_params={
  893. 'hostname': None,
  894. 'port': None,
  895. 'username': None,
  896. 'password': None,
  897. },
  898. borgmatic_runtime_directory='/run/borgmatic',
  899. )
  900. def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
  901. hook_config = [
  902. {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None},
  903. ]
  904. extract_process = flexmock(stdout=flexmock())
  905. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  906. 'resolve_credential',
  907. ).replace_with(lambda value, config: value)
  908. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  909. flexmock(module).should_receive('make_dump_path')
  910. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  911. flexmock(module).should_receive('execute_command_with_processes').with_args(
  912. (
  913. 'pg_restore',
  914. '--no-password',
  915. '--if-exists',
  916. '--exit-on-error',
  917. '--clean',
  918. '--dbname',
  919. 'foo',
  920. '--host',
  921. 'database.example.org',
  922. '--port',
  923. '5433',
  924. ),
  925. processes=[extract_process],
  926. output_log_level=logging.DEBUG,
  927. input_file=extract_process.stdout,
  928. environment={'PGSSLMODE': 'disable'},
  929. working_directory=None,
  930. ).once()
  931. flexmock(module).should_receive('execute_command').with_args(
  932. (
  933. 'psql',
  934. '--no-password',
  935. '--no-psqlrc',
  936. '--quiet',
  937. '--host',
  938. 'database.example.org',
  939. '--port',
  940. '5433',
  941. '--dbname',
  942. 'foo',
  943. '--command',
  944. 'ANALYZE',
  945. ),
  946. environment={'PGSSLMODE': 'disable'},
  947. working_directory=None,
  948. ).once()
  949. module.restore_data_source_dump(
  950. hook_config,
  951. {},
  952. data_source=hook_config[0],
  953. dry_run=False,
  954. extract_process=extract_process,
  955. connection_params={
  956. 'hostname': None,
  957. 'port': None,
  958. 'username': None,
  959. 'password': None,
  960. },
  961. borgmatic_runtime_directory='/run/borgmatic',
  962. )
  963. def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
  964. hook_config = [
  965. {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None},
  966. ]
  967. extract_process = flexmock(stdout=flexmock())
  968. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  969. 'resolve_credential',
  970. ).replace_with(lambda value, config: value)
  971. flexmock(module).should_receive('make_environment').and_return(
  972. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  973. )
  974. flexmock(module).should_receive('make_dump_path')
  975. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  976. flexmock(module).should_receive('execute_command_with_processes').with_args(
  977. (
  978. 'pg_restore',
  979. '--no-password',
  980. '--if-exists',
  981. '--exit-on-error',
  982. '--clean',
  983. '--dbname',
  984. 'foo',
  985. '--username',
  986. 'postgres',
  987. ),
  988. processes=[extract_process],
  989. output_log_level=logging.DEBUG,
  990. input_file=extract_process.stdout,
  991. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  992. working_directory=None,
  993. ).once()
  994. flexmock(module).should_receive('execute_command').with_args(
  995. (
  996. 'psql',
  997. '--no-password',
  998. '--no-psqlrc',
  999. '--quiet',
  1000. '--username',
  1001. 'postgres',
  1002. '--dbname',
  1003. 'foo',
  1004. '--command',
  1005. 'ANALYZE',
  1006. ),
  1007. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  1008. working_directory=None,
  1009. ).once()
  1010. module.restore_data_source_dump(
  1011. hook_config,
  1012. {},
  1013. data_source=hook_config[0],
  1014. dry_run=False,
  1015. extract_process=extract_process,
  1016. connection_params={
  1017. 'hostname': None,
  1018. 'port': None,
  1019. 'username': None,
  1020. 'password': None,
  1021. },
  1022. borgmatic_runtime_directory='/run/borgmatic',
  1023. )
  1024. def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
  1025. hook_config = [
  1026. {
  1027. 'name': 'foo',
  1028. 'hostname': 'database.example.org',
  1029. 'port': 5433,
  1030. 'username': 'postgres',
  1031. 'password': 'trustsome1',
  1032. 'restore_hostname': 'restorehost',
  1033. 'restore_port': 'restoreport',
  1034. 'restore_username': 'restoreusername',
  1035. 'restore_password': 'restorepassword',
  1036. 'schemas': None,
  1037. },
  1038. ]
  1039. extract_process = flexmock(stdout=flexmock())
  1040. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1041. 'resolve_credential',
  1042. ).replace_with(lambda value, config: value)
  1043. flexmock(module).should_receive('make_environment').and_return(
  1044. {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1045. )
  1046. flexmock(module).should_receive('make_dump_path')
  1047. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1048. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1049. (
  1050. 'pg_restore',
  1051. '--no-password',
  1052. '--if-exists',
  1053. '--exit-on-error',
  1054. '--clean',
  1055. '--dbname',
  1056. 'foo',
  1057. '--host',
  1058. 'clihost',
  1059. '--port',
  1060. 'cliport',
  1061. '--username',
  1062. 'cliusername',
  1063. ),
  1064. processes=[extract_process],
  1065. output_log_level=logging.DEBUG,
  1066. input_file=extract_process.stdout,
  1067. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1068. working_directory=None,
  1069. ).once()
  1070. flexmock(module).should_receive('execute_command').with_args(
  1071. (
  1072. 'psql',
  1073. '--no-password',
  1074. '--no-psqlrc',
  1075. '--quiet',
  1076. '--host',
  1077. 'clihost',
  1078. '--port',
  1079. 'cliport',
  1080. '--username',
  1081. 'cliusername',
  1082. '--dbname',
  1083. 'foo',
  1084. '--command',
  1085. 'ANALYZE',
  1086. ),
  1087. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1088. working_directory=None,
  1089. ).once()
  1090. module.restore_data_source_dump(
  1091. hook_config,
  1092. {},
  1093. data_source={'name': 'foo'},
  1094. dry_run=False,
  1095. extract_process=extract_process,
  1096. connection_params={
  1097. 'hostname': 'clihost',
  1098. 'port': 'cliport',
  1099. 'username': 'cliusername',
  1100. 'password': 'clipassword',
  1101. },
  1102. borgmatic_runtime_directory='/run/borgmatic',
  1103. )
  1104. def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore():
  1105. hook_config = [
  1106. {
  1107. 'name': 'foo',
  1108. 'hostname': 'database.example.org',
  1109. 'port': 5433,
  1110. 'username': 'postgres',
  1111. 'password': 'trustsome1',
  1112. 'schemas': None,
  1113. 'restore_hostname': 'restorehost',
  1114. 'restore_port': 'restoreport',
  1115. 'restore_username': 'restoreusername',
  1116. 'restore_password': 'restorepassword',
  1117. },
  1118. ]
  1119. extract_process = flexmock(stdout=flexmock())
  1120. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1121. 'resolve_credential',
  1122. ).replace_with(lambda value, config: value)
  1123. flexmock(module).should_receive('make_environment').and_return(
  1124. {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1125. )
  1126. flexmock(module).should_receive('make_dump_path')
  1127. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1128. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1129. (
  1130. 'pg_restore',
  1131. '--no-password',
  1132. '--if-exists',
  1133. '--exit-on-error',
  1134. '--clean',
  1135. '--dbname',
  1136. 'foo',
  1137. '--host',
  1138. 'restorehost',
  1139. '--port',
  1140. 'restoreport',
  1141. '--username',
  1142. 'restoreusername',
  1143. ),
  1144. processes=[extract_process],
  1145. output_log_level=logging.DEBUG,
  1146. input_file=extract_process.stdout,
  1147. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1148. working_directory=None,
  1149. ).once()
  1150. flexmock(module).should_receive('execute_command').with_args(
  1151. (
  1152. 'psql',
  1153. '--no-password',
  1154. '--no-psqlrc',
  1155. '--quiet',
  1156. '--host',
  1157. 'restorehost',
  1158. '--port',
  1159. 'restoreport',
  1160. '--username',
  1161. 'restoreusername',
  1162. '--dbname',
  1163. 'foo',
  1164. '--command',
  1165. 'ANALYZE',
  1166. ),
  1167. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1168. working_directory=None,
  1169. ).once()
  1170. module.restore_data_source_dump(
  1171. hook_config,
  1172. {},
  1173. data_source=hook_config[0],
  1174. dry_run=False,
  1175. extract_process=extract_process,
  1176. connection_params={
  1177. 'hostname': None,
  1178. 'port': None,
  1179. 'username': None,
  1180. 'password': None,
  1181. },
  1182. borgmatic_runtime_directory='/run/borgmatic',
  1183. )
  1184. def test_restore_data_source_dump_runs_pg_restore_with_options():
  1185. hook_config = [
  1186. {
  1187. 'name': 'foo',
  1188. 'restore_options': '--harder',
  1189. 'analyze_options': '--smarter',
  1190. 'schemas': None,
  1191. },
  1192. ]
  1193. extract_process = flexmock(stdout=flexmock())
  1194. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1195. 'resolve_credential',
  1196. ).replace_with(lambda value, config: value)
  1197. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1198. flexmock(module).should_receive('make_dump_path')
  1199. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1200. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1201. (
  1202. 'pg_restore',
  1203. '--no-password',
  1204. '--if-exists',
  1205. '--exit-on-error',
  1206. '--clean',
  1207. '--dbname',
  1208. 'foo',
  1209. '--harder',
  1210. ),
  1211. processes=[extract_process],
  1212. output_log_level=logging.DEBUG,
  1213. input_file=extract_process.stdout,
  1214. environment={'PGSSLMODE': 'disable'},
  1215. working_directory=None,
  1216. ).once()
  1217. flexmock(module).should_receive('execute_command').with_args(
  1218. (
  1219. 'psql',
  1220. '--no-password',
  1221. '--no-psqlrc',
  1222. '--quiet',
  1223. '--dbname',
  1224. 'foo',
  1225. '--smarter',
  1226. '--command',
  1227. 'ANALYZE',
  1228. ),
  1229. environment={'PGSSLMODE': 'disable'},
  1230. working_directory=None,
  1231. ).once()
  1232. module.restore_data_source_dump(
  1233. hook_config,
  1234. {},
  1235. data_source=hook_config[0],
  1236. dry_run=False,
  1237. extract_process=extract_process,
  1238. connection_params={
  1239. 'hostname': None,
  1240. 'port': None,
  1241. 'username': None,
  1242. 'password': None,
  1243. },
  1244. borgmatic_runtime_directory='/run/borgmatic',
  1245. )
  1246. def test_restore_data_source_dump_runs_psql_for_all_database_dump():
  1247. hook_config = [{'name': 'all', 'schemas': None}]
  1248. extract_process = flexmock(stdout=flexmock())
  1249. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1250. 'resolve_credential',
  1251. ).replace_with(lambda value, config: value)
  1252. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1253. flexmock(module).should_receive('make_dump_path')
  1254. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1255. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1256. (
  1257. 'psql',
  1258. '--no-password',
  1259. '--no-psqlrc',
  1260. ),
  1261. processes=[extract_process],
  1262. output_log_level=logging.DEBUG,
  1263. input_file=extract_process.stdout,
  1264. environment={'PGSSLMODE': 'disable'},
  1265. working_directory=None,
  1266. ).once()
  1267. flexmock(module).should_receive('execute_command').with_args(
  1268. ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
  1269. environment={'PGSSLMODE': 'disable'},
  1270. working_directory=None,
  1271. ).once()
  1272. module.restore_data_source_dump(
  1273. hook_config,
  1274. {},
  1275. data_source={'name': 'all'},
  1276. dry_run=False,
  1277. extract_process=extract_process,
  1278. connection_params={
  1279. 'hostname': None,
  1280. 'port': None,
  1281. 'username': None,
  1282. 'password': None,
  1283. },
  1284. borgmatic_runtime_directory='/run/borgmatic',
  1285. )
  1286. def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
  1287. hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
  1288. extract_process = flexmock(stdout=flexmock())
  1289. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1290. 'resolve_credential',
  1291. ).replace_with(lambda value, config: value)
  1292. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1293. flexmock(module).should_receive('make_dump_path')
  1294. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1295. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1296. ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
  1297. processes=[extract_process],
  1298. output_log_level=logging.DEBUG,
  1299. input_file=extract_process.stdout,
  1300. environment={'PGSSLMODE': 'disable'},
  1301. working_directory=None,
  1302. ).once()
  1303. flexmock(module).should_receive('execute_command').with_args(
  1304. (
  1305. 'psql',
  1306. '--no-password',
  1307. '--no-psqlrc',
  1308. '--quiet',
  1309. '--dbname',
  1310. 'foo',
  1311. '--command',
  1312. 'ANALYZE',
  1313. ),
  1314. environment={'PGSSLMODE': 'disable'},
  1315. working_directory=None,
  1316. ).once()
  1317. module.restore_data_source_dump(
  1318. hook_config,
  1319. {},
  1320. data_source=hook_config[0],
  1321. dry_run=False,
  1322. extract_process=extract_process,
  1323. connection_params={
  1324. 'hostname': None,
  1325. 'port': None,
  1326. 'username': None,
  1327. 'password': None,
  1328. },
  1329. borgmatic_runtime_directory='/run/borgmatic',
  1330. )
  1331. def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
  1332. hook_config = [
  1333. {
  1334. 'name': 'foo',
  1335. 'pg_restore_command': 'docker exec --workdir * mycontainer pg_restore',
  1336. 'psql_command': 'docker exec --workdir * mycontainer psql',
  1337. 'schemas': None,
  1338. },
  1339. ]
  1340. extract_process = flexmock(stdout=flexmock())
  1341. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1342. 'resolve_credential',
  1343. ).replace_with(lambda value, config: value)
  1344. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1345. flexmock(module).should_receive('make_dump_path')
  1346. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1347. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1348. (
  1349. 'docker',
  1350. 'exec',
  1351. '--workdir',
  1352. "'*'", # Should get shell escaped to prevent injection attacks.
  1353. 'mycontainer',
  1354. 'pg_restore',
  1355. '--no-password',
  1356. '--if-exists',
  1357. '--exit-on-error',
  1358. '--clean',
  1359. '--dbname',
  1360. 'foo',
  1361. ),
  1362. processes=[extract_process],
  1363. output_log_level=logging.DEBUG,
  1364. input_file=extract_process.stdout,
  1365. environment={'PGSSLMODE': 'disable'},
  1366. working_directory=None,
  1367. ).once()
  1368. flexmock(module).should_receive('execute_command').with_args(
  1369. (
  1370. 'docker',
  1371. 'exec',
  1372. '--workdir',
  1373. "'*'", # Should get shell escaped to prevent injection attacks.
  1374. 'mycontainer',
  1375. 'psql',
  1376. '--no-password',
  1377. '--no-psqlrc',
  1378. '--quiet',
  1379. '--dbname',
  1380. 'foo',
  1381. '--command',
  1382. 'ANALYZE',
  1383. ),
  1384. environment={'PGSSLMODE': 'disable'},
  1385. working_directory=None,
  1386. ).once()
  1387. module.restore_data_source_dump(
  1388. hook_config,
  1389. {},
  1390. data_source=hook_config[0],
  1391. dry_run=False,
  1392. extract_process=extract_process,
  1393. connection_params={
  1394. 'hostname': None,
  1395. 'port': None,
  1396. 'username': None,
  1397. 'password': None,
  1398. },
  1399. borgmatic_runtime_directory='/run/borgmatic',
  1400. )
  1401. def test_restore_data_source_dump_with_dry_run_skips_restore():
  1402. hook_config = [{'name': 'foo', 'schemas': None}]
  1403. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1404. 'resolve_credential',
  1405. ).replace_with(lambda value, config: value)
  1406. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1407. flexmock(module).should_receive('make_dump_path')
  1408. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1409. flexmock(module).should_receive('execute_command_with_processes').never()
  1410. module.restore_data_source_dump(
  1411. hook_config,
  1412. {},
  1413. data_source={'name': 'foo'},
  1414. dry_run=True,
  1415. extract_process=flexmock(),
  1416. connection_params={
  1417. 'hostname': None,
  1418. 'port': None,
  1419. 'username': None,
  1420. 'password': None,
  1421. },
  1422. borgmatic_runtime_directory='/run/borgmatic',
  1423. )
  1424. def test_restore_data_source_dump_without_extract_process_restores_from_disk():
  1425. hook_config = [{'name': 'foo', 'schemas': None}]
  1426. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1427. 'resolve_credential',
  1428. ).replace_with(lambda value, config: value)
  1429. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1430. flexmock(module).should_receive('make_dump_path')
  1431. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1432. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1433. (
  1434. 'pg_restore',
  1435. '--no-password',
  1436. '--if-exists',
  1437. '--exit-on-error',
  1438. '--clean',
  1439. '--dbname',
  1440. 'foo',
  1441. '/dump/path',
  1442. ),
  1443. processes=[],
  1444. output_log_level=logging.DEBUG,
  1445. input_file=None,
  1446. environment={'PGSSLMODE': 'disable'},
  1447. working_directory=None,
  1448. ).once()
  1449. flexmock(module).should_receive('execute_command').with_args(
  1450. (
  1451. 'psql',
  1452. '--no-password',
  1453. '--no-psqlrc',
  1454. '--quiet',
  1455. '--dbname',
  1456. 'foo',
  1457. '--command',
  1458. 'ANALYZE',
  1459. ),
  1460. environment={'PGSSLMODE': 'disable'},
  1461. working_directory=None,
  1462. ).once()
  1463. module.restore_data_source_dump(
  1464. hook_config,
  1465. {},
  1466. data_source={'name': 'foo'},
  1467. dry_run=False,
  1468. extract_process=None,
  1469. connection_params={
  1470. 'hostname': None,
  1471. 'port': None,
  1472. 'username': None,
  1473. 'password': None,
  1474. },
  1475. borgmatic_runtime_directory='/run/borgmatic',
  1476. )
  1477. def test_restore_data_source_dump_with_schemas_restores_schemas():
  1478. hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
  1479. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1480. 'resolve_credential',
  1481. ).replace_with(lambda value, config: value)
  1482. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1483. flexmock(module).should_receive('make_dump_path')
  1484. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1485. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1486. (
  1487. 'pg_restore',
  1488. '--no-password',
  1489. '--if-exists',
  1490. '--exit-on-error',
  1491. '--clean',
  1492. '--dbname',
  1493. 'foo',
  1494. '/dump/path',
  1495. '--schema',
  1496. 'bar',
  1497. '--schema',
  1498. 'baz',
  1499. ),
  1500. processes=[],
  1501. output_log_level=logging.DEBUG,
  1502. input_file=None,
  1503. environment={'PGSSLMODE': 'disable'},
  1504. working_directory=None,
  1505. ).once()
  1506. flexmock(module).should_receive('execute_command').with_args(
  1507. (
  1508. 'psql',
  1509. '--no-password',
  1510. '--no-psqlrc',
  1511. '--quiet',
  1512. '--dbname',
  1513. 'foo',
  1514. '--command',
  1515. 'ANALYZE',
  1516. ),
  1517. environment={'PGSSLMODE': 'disable'},
  1518. working_directory=None,
  1519. ).once()
  1520. module.restore_data_source_dump(
  1521. hook_config,
  1522. {},
  1523. data_source=hook_config[0],
  1524. dry_run=False,
  1525. extract_process=None,
  1526. connection_params={
  1527. 'hostname': None,
  1528. 'port': None,
  1529. 'username': None,
  1530. 'password': None,
  1531. },
  1532. borgmatic_runtime_directory='/run/borgmatic',
  1533. )