test_postgresql.py 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713
  1. import logging
  2. import pytest
  3. from flexmock import flexmock
  4. from borgmatic.hooks.data_source import postgresql as module
  5. def test_make_environment_maps_options_to_environment():
  6. database = {
  7. 'name': 'foo',
  8. 'password': 'pass',
  9. 'ssl_mode': 'require',
  10. 'ssl_cert': 'cert.crt',
  11. 'ssl_key': 'key.key',
  12. 'ssl_root_cert': 'root.crt',
  13. 'ssl_crl': 'crl.crl',
  14. }
  15. expected = {
  16. 'USER': 'root',
  17. 'PGPASSWORD': 'pass',
  18. 'PGSSLMODE': 'require',
  19. 'PGSSLCERT': 'cert.crt',
  20. 'PGSSLKEY': 'key.key',
  21. 'PGSSLROOTCERT': 'root.crt',
  22. 'PGSSLCRL': 'crl.crl',
  23. }
  24. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  25. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  26. 'resolve_credential',
  27. ).replace_with(lambda value, config: value)
  28. assert module.make_environment(database, {}) == expected
  29. def test_make_environment_with_cli_password_sets_correct_password():
  30. database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
  31. flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
  32. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  33. 'resolve_credential',
  34. ).replace_with(lambda value, config: value)
  35. environment = module.make_environment(
  36. database,
  37. {},
  38. restore_connection_params={'password': 'clipassword'},
  39. )
  40. assert environment['PGPASSWORD'] == 'clipassword'
  41. def test_make_environment_without_cli_password_or_configured_password_does_not_set_password():
  42. database = {'name': 'foo'}
  43. environment = module.make_environment(
  44. database,
  45. {},
  46. restore_connection_params={'username': 'someone'},
  47. )
  48. assert 'PGPASSWORD' not in environment
  49. def test_make_environment_without_ssl_mode_does_not_set_ssl_mode():
  50. database = {'name': 'foo'}
  51. environment = module.make_environment(database, {})
  52. assert 'PGSSLMODE' not in environment
  53. def test_database_names_to_dump_passes_through_individual_database_name():
  54. database = {'name': 'foo'}
  55. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  56. def test_database_names_to_dump_passes_through_individual_database_name_with_format():
  57. database = {'name': 'foo', 'format': 'custom'}
  58. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  59. def test_database_names_to_dump_passes_through_all_without_format():
  60. database = {'name': 'all'}
  61. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('all',)
  62. def test_database_names_to_dump_with_all_and_format_and_dry_run_bails():
  63. database = {'name': 'all', 'format': 'custom'}
  64. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  65. 'resolve_credential',
  66. ).replace_with(lambda value, config: value)
  67. flexmock(module).should_receive('execute_command_and_capture_output').never()
  68. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=True) == ()
  69. def test_database_names_to_dump_with_all_and_format_lists_databases():
  70. database = {'name': 'all', 'format': 'custom'}
  71. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  72. 'resolve_credential',
  73. ).replace_with(lambda value, config: value)
  74. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  75. 'foo,test,\nbar,test,"stuff and such"',
  76. )
  77. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  78. 'foo',
  79. 'bar',
  80. )
  81. def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
  82. database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
  83. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  84. 'resolve_credential',
  85. ).replace_with(lambda value, config: value)
  86. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
  87. '/path/to/working/dir'
  88. )
  89. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  90. (
  91. 'psql',
  92. '--list',
  93. '--no-password',
  94. '--no-psqlrc',
  95. '--csv',
  96. '--tuples-only',
  97. '--host',
  98. 'localhost',
  99. '--port',
  100. '1234',
  101. ),
  102. environment=object,
  103. working_directory='/path/to/working/dir',
  104. ).and_return('foo,test,\nbar,test,"stuff and such"')
  105. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  106. 'foo',
  107. 'bar',
  108. )
  109. def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
  110. database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
  111. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  112. 'resolve_credential',
  113. ).replace_with(lambda value, config: value)
  114. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  115. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  116. (
  117. 'psql',
  118. '--list',
  119. '--no-password',
  120. '--no-psqlrc',
  121. '--csv',
  122. '--tuples-only',
  123. '--username',
  124. 'postgres',
  125. ),
  126. environment=object,
  127. working_directory=None,
  128. ).and_return('foo,test,\nbar,test,"stuff and such"')
  129. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  130. 'foo',
  131. 'bar',
  132. )
  133. def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
  134. database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
  135. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  136. 'resolve_credential',
  137. ).replace_with(lambda value, config: value)
  138. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  139. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  140. ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
  141. environment=object,
  142. working_directory=None,
  143. ).and_return('foo,test,\nbar,test,"stuff and such"')
  144. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
  145. 'foo',
  146. 'bar',
  147. )
  148. def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
  149. database = {'name': 'all', 'format': 'custom'}
  150. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  151. 'resolve_credential',
  152. ).replace_with(lambda value, config: value)
  153. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  154. 'foo,test,\ntemplate0,test,blah',
  155. )
  156. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  157. def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
  158. database = {
  159. 'name': 'all',
  160. 'format': 'custom',
  161. 'psql_command': 'docker exec --workdir * mycontainer psql',
  162. }
  163. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  164. 'resolve_credential',
  165. ).replace_with(lambda value, config: value)
  166. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  167. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  168. (
  169. 'docker',
  170. 'exec',
  171. '--workdir',
  172. "'*'", # Should get shell escaped to prevent injection attacks.
  173. 'mycontainer',
  174. 'psql',
  175. '--list',
  176. '--no-password',
  177. '--no-psqlrc',
  178. '--csv',
  179. '--tuples-only',
  180. ),
  181. environment=object,
  182. working_directory=None,
  183. ).and_return('foo,text').once()
  184. assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
  185. def test_use_streaming_true_for_any_non_directory_format_databases():
  186. assert module.use_streaming(
  187. databases=[{'format': 'stuff'}, {'format': 'directory'}, {}],
  188. config=flexmock(),
  189. )
  190. def test_use_streaming_false_for_all_directory_format_databases():
  191. assert not module.use_streaming(
  192. databases=[{'format': 'directory'}, {'format': 'directory'}],
  193. config=flexmock(),
  194. )
  195. def test_use_streaming_false_for_no_databases():
  196. assert not module.use_streaming(databases=[], config=flexmock())
  197. def test_dump_data_sources_runs_pg_dump_for_each_database():
  198. databases = [{'name': 'foo'}, {'name': 'bar'}]
  199. processes = [flexmock(), flexmock()]
  200. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  201. flexmock(module).should_receive('make_dump_path').and_return('')
  202. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  203. ('bar',),
  204. )
  205. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  206. 'databases/localhost/foo',
  207. ).and_return('databases/localhost/bar')
  208. flexmock(module.os.path).should_receive('exists').and_return(False)
  209. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  210. 'resolve_credential',
  211. ).replace_with(lambda value, config: value)
  212. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  213. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  214. for name, process in zip(('foo', 'bar'), processes):
  215. flexmock(module).should_receive('execute_command').with_args(
  216. (
  217. 'pg_dump',
  218. '--no-password',
  219. '--clean',
  220. '--if-exists',
  221. '--format',
  222. 'custom',
  223. name,
  224. '>',
  225. f'databases/localhost/{name}',
  226. ),
  227. shell=True,
  228. environment={'PGSSLMODE': 'disable'},
  229. run_to_completion=False,
  230. working_directory=None,
  231. ).and_return(process).once()
  232. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  233. '/run/borgmatic',
  234. 'postgresql_databases',
  235. [
  236. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  237. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  238. ],
  239. ).once()
  240. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  241. object,
  242. module.borgmatic.borg.pattern.Pattern(
  243. '/run/borgmatic/postgresql_databases',
  244. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  245. ),
  246. ).once()
  247. assert (
  248. module.dump_data_sources(
  249. databases,
  250. {},
  251. config_paths=('test.yaml',),
  252. borgmatic_runtime_directory='/run/borgmatic',
  253. patterns=[],
  254. dry_run=False,
  255. )
  256. == processes
  257. )
  258. def test_dump_data_sources_raises_when_no_database_names_to_dump():
  259. databases = [{'name': 'foo'}, {'name': 'bar'}]
  260. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  261. flexmock(module).should_receive('make_dump_path').and_return('')
  262. flexmock(module).should_receive('database_names_to_dump').and_return(())
  263. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  264. with pytest.raises(ValueError):
  265. module.dump_data_sources(
  266. databases,
  267. {},
  268. config_paths=('test.yaml',),
  269. borgmatic_runtime_directory='/run/borgmatic',
  270. patterns=[],
  271. dry_run=False,
  272. )
  273. def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
  274. databases = [{'name': 'foo'}, {'name': 'bar'}]
  275. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  276. flexmock(module).should_receive('make_dump_path').and_return('')
  277. flexmock(module).should_receive('database_names_to_dump').and_return(())
  278. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  279. assert (
  280. module.dump_data_sources(
  281. databases,
  282. {},
  283. config_paths=('test.yaml',),
  284. borgmatic_runtime_directory='/run/borgmatic',
  285. patterns=[],
  286. dry_run=True,
  287. )
  288. == []
  289. )
  290. def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
  291. databases = [{'name': 'foo'}, {'name': 'bar'}]
  292. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  293. flexmock(module).should_receive('make_dump_path').and_return('')
  294. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  295. ('bar',),
  296. )
  297. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  298. 'databases/localhost/foo',
  299. ).and_return('databases/localhost/bar')
  300. flexmock(module.os.path).should_receive('exists').and_return(True)
  301. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  302. flexmock(module).should_receive('execute_command').never()
  303. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  304. '/run/borgmatic',
  305. 'postgresql_databases',
  306. [
  307. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  308. module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
  309. ],
  310. ).once()
  311. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  312. object,
  313. module.borgmatic.borg.pattern.Pattern(
  314. '/run/borgmatic/postgresql_databases',
  315. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  316. ),
  317. ).once()
  318. assert (
  319. module.dump_data_sources(
  320. databases,
  321. {},
  322. config_paths=('test.yaml',),
  323. borgmatic_runtime_directory='/run/borgmatic',
  324. patterns=[],
  325. dry_run=False,
  326. )
  327. == []
  328. )
  329. def test_dump_data_sources_with_dry_run_skips_pg_dump():
  330. databases = [{'name': 'foo'}, {'name': 'bar'}]
  331. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  332. flexmock(module).should_receive('make_dump_path').and_return('')
  333. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  334. ('bar',),
  335. )
  336. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  337. 'databases/localhost/foo',
  338. ).and_return('databases/localhost/bar')
  339. flexmock(module.os.path).should_receive('exists').and_return(False)
  340. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  341. 'resolve_credential',
  342. ).replace_with(lambda value, config: value)
  343. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  344. flexmock(module).should_receive('execute_command').never()
  345. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
  346. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
  347. assert (
  348. module.dump_data_sources(
  349. databases,
  350. {},
  351. config_paths=('test.yaml',),
  352. borgmatic_runtime_directory='/run/borgmatic',
  353. patterns=[],
  354. dry_run=True,
  355. )
  356. == []
  357. )
  358. def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
  359. databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
  360. process = flexmock()
  361. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  362. flexmock(module).should_receive('make_dump_path').and_return('')
  363. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  364. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  365. 'databases/database.example.org/foo',
  366. )
  367. flexmock(module.os.path).should_receive('exists').and_return(False)
  368. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  369. 'resolve_credential',
  370. ).replace_with(lambda value, config: value)
  371. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  372. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  373. flexmock(module).should_receive('execute_command').with_args(
  374. (
  375. 'pg_dump',
  376. '--no-password',
  377. '--clean',
  378. '--if-exists',
  379. '--host',
  380. 'database.example.org',
  381. '--port',
  382. '5433',
  383. '--format',
  384. 'custom',
  385. 'foo',
  386. '>',
  387. 'databases/database.example.org/foo',
  388. ),
  389. shell=True,
  390. environment={'PGSSLMODE': 'disable'},
  391. run_to_completion=False,
  392. working_directory=None,
  393. ).and_return(process).once()
  394. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  395. '/run/borgmatic',
  396. 'postgresql_databases',
  397. [
  398. module.borgmatic.actions.restore.Dump(
  399. 'postgresql_databases', 'foo', 'database.example.org', 5433
  400. ),
  401. ],
  402. ).once()
  403. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  404. object,
  405. module.borgmatic.borg.pattern.Pattern(
  406. '/run/borgmatic/postgresql_databases',
  407. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  408. ),
  409. ).once()
  410. assert module.dump_data_sources(
  411. databases,
  412. {},
  413. config_paths=('test.yaml',),
  414. borgmatic_runtime_directory='/run/borgmatic',
  415. patterns=[],
  416. dry_run=False,
  417. ) == [process]
  418. def test_dump_data_sources_runs_pg_dump_with_username_and_password():
  419. databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
  420. process = flexmock()
  421. flexmock(module).should_receive('make_environment').and_return(
  422. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  423. )
  424. flexmock(module).should_receive('make_dump_path').and_return('')
  425. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  426. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  427. 'databases/localhost/foo',
  428. )
  429. flexmock(module.os.path).should_receive('exists').and_return(False)
  430. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  431. 'resolve_credential',
  432. ).replace_with(lambda value, config: value)
  433. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  434. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  435. flexmock(module).should_receive('execute_command').with_args(
  436. (
  437. 'pg_dump',
  438. '--no-password',
  439. '--clean',
  440. '--if-exists',
  441. '--username',
  442. 'postgres',
  443. '--format',
  444. 'custom',
  445. 'foo',
  446. '>',
  447. 'databases/localhost/foo',
  448. ),
  449. shell=True,
  450. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  451. run_to_completion=False,
  452. working_directory=None,
  453. ).and_return(process).once()
  454. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  455. '/run/borgmatic',
  456. 'postgresql_databases',
  457. [
  458. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  459. ],
  460. ).once()
  461. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  462. object,
  463. module.borgmatic.borg.pattern.Pattern(
  464. '/run/borgmatic/postgresql_databases',
  465. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  466. ),
  467. ).once()
  468. assert module.dump_data_sources(
  469. databases,
  470. {},
  471. config_paths=('test.yaml',),
  472. borgmatic_runtime_directory='/run/borgmatic',
  473. patterns=[],
  474. dry_run=False,
  475. ) == [process]
  476. def test_dump_data_sources_with_username_injection_attack_gets_escaped():
  477. databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}]
  478. process = flexmock()
  479. flexmock(module).should_receive('make_environment').and_return(
  480. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  481. )
  482. flexmock(module).should_receive('make_dump_path').and_return('')
  483. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  484. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  485. 'databases/localhost/foo',
  486. )
  487. flexmock(module.os.path).should_receive('exists').and_return(False)
  488. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  489. 'resolve_credential',
  490. ).replace_with(lambda value, config: value)
  491. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  492. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  493. flexmock(module).should_receive('execute_command').with_args(
  494. (
  495. 'pg_dump',
  496. '--no-password',
  497. '--clean',
  498. '--if-exists',
  499. '--username',
  500. "'postgres; naughty-command'",
  501. '--format',
  502. 'custom',
  503. 'foo',
  504. '>',
  505. 'databases/localhost/foo',
  506. ),
  507. shell=True,
  508. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  509. run_to_completion=False,
  510. working_directory=None,
  511. ).and_return(process).once()
  512. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  513. '/run/borgmatic',
  514. 'postgresql_databases',
  515. [
  516. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  517. ],
  518. ).once()
  519. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  520. object,
  521. module.borgmatic.borg.pattern.Pattern(
  522. '/run/borgmatic/postgresql_databases',
  523. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  524. ),
  525. ).once()
  526. assert module.dump_data_sources(
  527. databases,
  528. {},
  529. config_paths=('test.yaml',),
  530. borgmatic_runtime_directory='/run/borgmatic',
  531. patterns=[],
  532. dry_run=False,
  533. ) == [process]
  534. def test_dump_data_sources_runs_pg_dump_with_directory_format():
  535. databases = [{'name': 'foo', 'format': 'directory'}]
  536. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  537. flexmock(module).should_receive('make_dump_path').and_return('')
  538. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  539. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  540. 'databases/localhost/foo',
  541. )
  542. flexmock(module.os.path).should_receive('exists').and_return(False)
  543. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  544. 'resolve_credential',
  545. ).replace_with(lambda value, config: value)
  546. flexmock(module.dump).should_receive('create_parent_directory_for_dump')
  547. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  548. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  549. flexmock(module).should_receive('execute_command').with_args(
  550. (
  551. 'pg_dump',
  552. '--no-password',
  553. '--clean',
  554. '--if-exists',
  555. '--format',
  556. 'directory',
  557. '--file',
  558. 'databases/localhost/foo',
  559. 'foo',
  560. ),
  561. shell=True,
  562. environment={'PGSSLMODE': 'disable'},
  563. working_directory=None,
  564. ).and_return(flexmock()).once()
  565. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  566. '/run/borgmatic',
  567. 'postgresql_databases',
  568. [
  569. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  570. ],
  571. ).once()
  572. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  573. object,
  574. module.borgmatic.borg.pattern.Pattern(
  575. '/run/borgmatic/postgresql_databases',
  576. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  577. ),
  578. ).once()
  579. assert (
  580. module.dump_data_sources(
  581. databases,
  582. {},
  583. config_paths=('test.yaml',),
  584. borgmatic_runtime_directory='/run/borgmatic',
  585. patterns=[],
  586. dry_run=False,
  587. )
  588. == []
  589. )
  590. def test_dump_data_sources_runs_pg_dump_with_string_compression():
  591. databases = [{'name': 'foo', 'compression': 'winrar'}]
  592. processes = [flexmock()]
  593. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  594. flexmock(module).should_receive('make_dump_path').and_return('')
  595. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  596. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  597. 'databases/localhost/foo',
  598. )
  599. flexmock(module.os.path).should_receive('exists').and_return(False)
  600. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  601. 'resolve_credential',
  602. ).replace_with(lambda value, config: value)
  603. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  604. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  605. flexmock(module).should_receive('execute_command').with_args(
  606. (
  607. 'pg_dump',
  608. '--no-password',
  609. '--clean',
  610. '--if-exists',
  611. '--format',
  612. 'custom',
  613. '--compress',
  614. 'winrar',
  615. 'foo',
  616. '>',
  617. 'databases/localhost/foo',
  618. ),
  619. shell=True,
  620. environment={'PGSSLMODE': 'disable'},
  621. run_to_completion=False,
  622. working_directory=None,
  623. ).and_return(processes[0]).once()
  624. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  625. '/run/borgmatic',
  626. 'postgresql_databases',
  627. [
  628. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  629. ],
  630. ).once()
  631. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  632. object,
  633. module.borgmatic.borg.pattern.Pattern(
  634. '/run/borgmatic/postgresql_databases',
  635. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  636. ),
  637. ).once()
  638. assert (
  639. module.dump_data_sources(
  640. databases,
  641. {},
  642. config_paths=('test.yaml',),
  643. borgmatic_runtime_directory='/run/borgmatic',
  644. patterns=[],
  645. dry_run=False,
  646. )
  647. == processes
  648. )
  649. def test_dump_data_sources_runs_pg_dump_with_integer_compression():
  650. databases = [{'name': 'foo', 'compression': 0}]
  651. processes = [flexmock()]
  652. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  653. flexmock(module).should_receive('make_dump_path').and_return('')
  654. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  655. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  656. 'databases/localhost/foo',
  657. )
  658. flexmock(module.os.path).should_receive('exists').and_return(False)
  659. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  660. 'resolve_credential',
  661. ).replace_with(lambda value, config: value)
  662. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  663. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  664. flexmock(module).should_receive('execute_command').with_args(
  665. (
  666. 'pg_dump',
  667. '--no-password',
  668. '--clean',
  669. '--if-exists',
  670. '--format',
  671. 'custom',
  672. '--compress',
  673. '0',
  674. 'foo',
  675. '>',
  676. 'databases/localhost/foo',
  677. ),
  678. shell=True,
  679. environment={'PGSSLMODE': 'disable'},
  680. run_to_completion=False,
  681. working_directory=None,
  682. ).and_return(processes[0]).once()
  683. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  684. '/run/borgmatic',
  685. 'postgresql_databases',
  686. [
  687. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  688. ],
  689. ).once()
  690. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  691. object,
  692. module.borgmatic.borg.pattern.Pattern(
  693. '/run/borgmatic/postgresql_databases',
  694. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  695. ),
  696. ).once()
  697. assert (
  698. module.dump_data_sources(
  699. databases,
  700. {},
  701. config_paths=('test.yaml',),
  702. borgmatic_runtime_directory='/run/borgmatic',
  703. patterns=[],
  704. dry_run=False,
  705. )
  706. == processes
  707. )
  708. def test_dump_data_sources_runs_pg_dump_with_options():
  709. databases = [{'name': 'foo', 'options': '--stuff=such'}]
  710. process = flexmock()
  711. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  712. flexmock(module).should_receive('make_dump_path').and_return('')
  713. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  714. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  715. 'databases/localhost/foo',
  716. )
  717. flexmock(module.os.path).should_receive('exists').and_return(False)
  718. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  719. 'resolve_credential',
  720. ).replace_with(lambda value, config: value)
  721. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  722. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  723. flexmock(module).should_receive('execute_command').with_args(
  724. (
  725. 'pg_dump',
  726. '--no-password',
  727. '--clean',
  728. '--if-exists',
  729. '--format',
  730. 'custom',
  731. '--stuff=such',
  732. 'foo',
  733. '>',
  734. 'databases/localhost/foo',
  735. ),
  736. shell=True,
  737. environment={'PGSSLMODE': 'disable'},
  738. run_to_completion=False,
  739. working_directory=None,
  740. ).and_return(process).once()
  741. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  742. '/run/borgmatic',
  743. 'postgresql_databases',
  744. [
  745. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  746. ],
  747. ).once()
  748. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  749. object,
  750. module.borgmatic.borg.pattern.Pattern(
  751. '/run/borgmatic/postgresql_databases',
  752. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  753. ),
  754. ).once()
  755. assert module.dump_data_sources(
  756. databases,
  757. {},
  758. config_paths=('test.yaml',),
  759. borgmatic_runtime_directory='/run/borgmatic',
  760. patterns=[],
  761. dry_run=False,
  762. ) == [process]
  763. def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
  764. databases = [{'name': 'all'}]
  765. process = flexmock()
  766. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  767. flexmock(module).should_receive('make_dump_path').and_return('')
  768. flexmock(module).should_receive('database_names_to_dump').and_return(('all',))
  769. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  770. 'databases/localhost/all',
  771. )
  772. flexmock(module.os.path).should_receive('exists').and_return(False)
  773. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  774. 'resolve_credential',
  775. ).replace_with(lambda value, config: value)
  776. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  777. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  778. flexmock(module).should_receive('execute_command').with_args(
  779. ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
  780. shell=True,
  781. environment={'PGSSLMODE': 'disable'},
  782. run_to_completion=False,
  783. working_directory=None,
  784. ).and_return(process).once()
  785. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  786. '/run/borgmatic',
  787. 'postgresql_databases',
  788. [
  789. module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'),
  790. ],
  791. ).once()
  792. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  793. object,
  794. module.borgmatic.borg.pattern.Pattern(
  795. '/run/borgmatic/postgresql_databases',
  796. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  797. ),
  798. ).once()
  799. assert module.dump_data_sources(
  800. databases,
  801. {},
  802. config_paths=('test.yaml',),
  803. borgmatic_runtime_directory='/run/borgmatic',
  804. patterns=[],
  805. dry_run=False,
  806. ) == [process]
  807. def test_dump_data_sources_runs_non_default_pg_dump():
  808. databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}]
  809. process = flexmock()
  810. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  811. flexmock(module).should_receive('make_dump_path').and_return('')
  812. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  813. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
  814. 'databases/localhost/foo',
  815. )
  816. flexmock(module.os.path).should_receive('exists').and_return(False)
  817. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  818. 'resolve_credential',
  819. ).replace_with(lambda value, config: value)
  820. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  821. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  822. flexmock(module).should_receive('execute_command').with_args(
  823. (
  824. 'special_pg_dump',
  825. '--compress',
  826. "'*'", # Should get shell escaped to prevent injection attacks.
  827. '--no-password',
  828. '--clean',
  829. '--if-exists',
  830. '--format',
  831. 'custom',
  832. 'foo',
  833. '>',
  834. 'databases/localhost/foo',
  835. ),
  836. shell=True,
  837. environment={'PGSSLMODE': 'disable'},
  838. run_to_completion=False,
  839. working_directory=None,
  840. ).and_return(process).once()
  841. flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
  842. '/run/borgmatic',
  843. 'postgresql_databases',
  844. [
  845. module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
  846. ],
  847. ).once()
  848. flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
  849. object,
  850. module.borgmatic.borg.pattern.Pattern(
  851. '/run/borgmatic/postgresql_databases',
  852. source=module.borgmatic.borg.pattern.Pattern_source.HOOK,
  853. ),
  854. ).once()
  855. assert module.dump_data_sources(
  856. databases,
  857. {},
  858. config_paths=('test.yaml',),
  859. borgmatic_runtime_directory='/run/borgmatic',
  860. patterns=[],
  861. dry_run=False,
  862. ) == [process]
  863. def test_restore_data_source_dump_runs_pg_restore():
  864. hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}]
  865. extract_process = flexmock(stdout=flexmock())
  866. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  867. 'resolve_credential',
  868. ).replace_with(lambda value, config: value)
  869. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  870. flexmock(module).should_receive('make_dump_path')
  871. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  872. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  873. flexmock(module).should_receive('execute_command_with_processes').with_args(
  874. (
  875. 'pg_restore',
  876. '--no-password',
  877. '--if-exists',
  878. '--exit-on-error',
  879. '--clean',
  880. '--dbname',
  881. 'foo',
  882. ),
  883. processes=[extract_process],
  884. output_log_level=logging.DEBUG,
  885. input_file=extract_process.stdout,
  886. environment={'PGSSLMODE': 'disable'},
  887. working_directory=None,
  888. ).once()
  889. flexmock(module).should_receive('execute_command').with_args(
  890. (
  891. 'psql',
  892. '--no-password',
  893. '--no-psqlrc',
  894. '--quiet',
  895. '--dbname',
  896. 'foo',
  897. '--command',
  898. 'ANALYZE',
  899. ),
  900. environment={'PGSSLMODE': 'disable'},
  901. working_directory=None,
  902. ).once()
  903. module.restore_data_source_dump(
  904. hook_config,
  905. {},
  906. data_source={'name': 'foo'},
  907. dry_run=False,
  908. extract_process=extract_process,
  909. connection_params={
  910. 'hostname': None,
  911. 'port': None,
  912. 'username': None,
  913. 'password': None,
  914. },
  915. borgmatic_runtime_directory='/run/borgmatic',
  916. )
  917. def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
  918. hook_config = [
  919. {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None},
  920. ]
  921. extract_process = flexmock(stdout=flexmock())
  922. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  923. 'resolve_credential',
  924. ).replace_with(lambda value, config: value)
  925. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  926. flexmock(module).should_receive('make_dump_path')
  927. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  928. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  929. flexmock(module).should_receive('execute_command_with_processes').with_args(
  930. (
  931. 'pg_restore',
  932. '--no-password',
  933. '--if-exists',
  934. '--exit-on-error',
  935. '--clean',
  936. '--dbname',
  937. 'foo',
  938. '--host',
  939. 'database.example.org',
  940. '--port',
  941. '5433',
  942. ),
  943. processes=[extract_process],
  944. output_log_level=logging.DEBUG,
  945. input_file=extract_process.stdout,
  946. environment={'PGSSLMODE': 'disable'},
  947. working_directory=None,
  948. ).once()
  949. flexmock(module).should_receive('execute_command').with_args(
  950. (
  951. 'psql',
  952. '--no-password',
  953. '--no-psqlrc',
  954. '--quiet',
  955. '--host',
  956. 'database.example.org',
  957. '--port',
  958. '5433',
  959. '--dbname',
  960. 'foo',
  961. '--command',
  962. 'ANALYZE',
  963. ),
  964. environment={'PGSSLMODE': 'disable'},
  965. working_directory=None,
  966. ).once()
  967. module.restore_data_source_dump(
  968. hook_config,
  969. {},
  970. data_source=hook_config[0],
  971. dry_run=False,
  972. extract_process=extract_process,
  973. connection_params={
  974. 'hostname': None,
  975. 'port': None,
  976. 'username': None,
  977. 'password': None,
  978. },
  979. borgmatic_runtime_directory='/run/borgmatic',
  980. )
  981. def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
  982. hook_config = [
  983. {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None},
  984. ]
  985. extract_process = flexmock(stdout=flexmock())
  986. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  987. 'resolve_credential',
  988. ).replace_with(lambda value, config: value)
  989. flexmock(module).should_receive('make_environment').and_return(
  990. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  991. )
  992. flexmock(module).should_receive('make_dump_path')
  993. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  994. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  995. flexmock(module).should_receive('execute_command_with_processes').with_args(
  996. (
  997. 'pg_restore',
  998. '--no-password',
  999. '--if-exists',
  1000. '--exit-on-error',
  1001. '--clean',
  1002. '--dbname',
  1003. 'foo',
  1004. '--username',
  1005. 'postgres',
  1006. ),
  1007. processes=[extract_process],
  1008. output_log_level=logging.DEBUG,
  1009. input_file=extract_process.stdout,
  1010. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  1011. working_directory=None,
  1012. ).once()
  1013. flexmock(module).should_receive('execute_command').with_args(
  1014. (
  1015. 'psql',
  1016. '--no-password',
  1017. '--no-psqlrc',
  1018. '--quiet',
  1019. '--username',
  1020. 'postgres',
  1021. '--dbname',
  1022. 'foo',
  1023. '--command',
  1024. 'ANALYZE',
  1025. ),
  1026. environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  1027. working_directory=None,
  1028. ).once()
  1029. module.restore_data_source_dump(
  1030. hook_config,
  1031. {},
  1032. data_source=hook_config[0],
  1033. dry_run=False,
  1034. extract_process=extract_process,
  1035. connection_params={
  1036. 'hostname': None,
  1037. 'port': None,
  1038. 'username': None,
  1039. 'password': None,
  1040. },
  1041. borgmatic_runtime_directory='/run/borgmatic',
  1042. )
  1043. def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore():
  1044. hook_config = [
  1045. {
  1046. 'name': 'foo',
  1047. 'hostname': 'database.example.org',
  1048. 'port': 5433,
  1049. 'username': 'postgres',
  1050. 'password': 'trustsome1',
  1051. 'restore_hostname': 'restorehost',
  1052. 'restore_port': 'restoreport',
  1053. 'restore_username': 'restoreusername',
  1054. 'restore_password': 'restorepassword',
  1055. 'schemas': None,
  1056. },
  1057. ]
  1058. extract_process = flexmock(stdout=flexmock())
  1059. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1060. 'resolve_credential',
  1061. ).replace_with(lambda value, config: value)
  1062. flexmock(module).should_receive('make_environment').and_return(
  1063. {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1064. )
  1065. flexmock(module).should_receive('make_dump_path')
  1066. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1067. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1068. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1069. (
  1070. 'pg_restore',
  1071. '--no-password',
  1072. '--if-exists',
  1073. '--exit-on-error',
  1074. '--clean',
  1075. '--dbname',
  1076. 'foo',
  1077. '--host',
  1078. 'clihost',
  1079. '--port',
  1080. 'cliport',
  1081. '--username',
  1082. 'cliusername',
  1083. ),
  1084. processes=[extract_process],
  1085. output_log_level=logging.DEBUG,
  1086. input_file=extract_process.stdout,
  1087. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1088. working_directory=None,
  1089. ).once()
  1090. flexmock(module).should_receive('execute_command').with_args(
  1091. (
  1092. 'psql',
  1093. '--no-password',
  1094. '--no-psqlrc',
  1095. '--quiet',
  1096. '--host',
  1097. 'clihost',
  1098. '--port',
  1099. 'cliport',
  1100. '--username',
  1101. 'cliusername',
  1102. '--dbname',
  1103. 'foo',
  1104. '--command',
  1105. 'ANALYZE',
  1106. ),
  1107. environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
  1108. working_directory=None,
  1109. ).once()
  1110. module.restore_data_source_dump(
  1111. hook_config,
  1112. {},
  1113. data_source={'name': 'foo'},
  1114. dry_run=False,
  1115. extract_process=extract_process,
  1116. connection_params={
  1117. 'hostname': 'clihost',
  1118. 'port': 'cliport',
  1119. 'username': 'cliusername',
  1120. 'password': 'clipassword',
  1121. },
  1122. borgmatic_runtime_directory='/run/borgmatic',
  1123. )
  1124. def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore():
  1125. hook_config = [
  1126. {
  1127. 'name': 'foo',
  1128. 'hostname': 'database.example.org',
  1129. 'port': 5433,
  1130. 'username': 'postgres',
  1131. 'password': 'trustsome1',
  1132. 'schemas': None,
  1133. 'restore_hostname': 'restorehost',
  1134. 'restore_port': 'restoreport',
  1135. 'restore_username': 'restoreusername',
  1136. 'restore_password': 'restorepassword',
  1137. },
  1138. ]
  1139. extract_process = flexmock(stdout=flexmock())
  1140. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1141. 'resolve_credential',
  1142. ).replace_with(lambda value, config: value)
  1143. flexmock(module).should_receive('make_environment').and_return(
  1144. {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1145. )
  1146. flexmock(module).should_receive('make_dump_path')
  1147. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1148. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1149. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1150. (
  1151. 'pg_restore',
  1152. '--no-password',
  1153. '--if-exists',
  1154. '--exit-on-error',
  1155. '--clean',
  1156. '--dbname',
  1157. 'foo',
  1158. '--host',
  1159. 'restorehost',
  1160. '--port',
  1161. 'restoreport',
  1162. '--username',
  1163. 'restoreusername',
  1164. ),
  1165. processes=[extract_process],
  1166. output_log_level=logging.DEBUG,
  1167. input_file=extract_process.stdout,
  1168. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1169. working_directory=None,
  1170. ).once()
  1171. flexmock(module).should_receive('execute_command').with_args(
  1172. (
  1173. 'psql',
  1174. '--no-password',
  1175. '--no-psqlrc',
  1176. '--quiet',
  1177. '--host',
  1178. 'restorehost',
  1179. '--port',
  1180. 'restoreport',
  1181. '--username',
  1182. 'restoreusername',
  1183. '--dbname',
  1184. 'foo',
  1185. '--command',
  1186. 'ANALYZE',
  1187. ),
  1188. environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
  1189. working_directory=None,
  1190. ).once()
  1191. module.restore_data_source_dump(
  1192. hook_config,
  1193. {},
  1194. data_source=hook_config[0],
  1195. dry_run=False,
  1196. extract_process=extract_process,
  1197. connection_params={
  1198. 'hostname': None,
  1199. 'port': None,
  1200. 'username': None,
  1201. 'password': None,
  1202. },
  1203. borgmatic_runtime_directory='/run/borgmatic',
  1204. )
  1205. def test_restore_data_source_dump_runs_pg_restore_with_options():
  1206. hook_config = [
  1207. {
  1208. 'name': 'foo',
  1209. 'restore_options': '--harder',
  1210. 'analyze_options': '--smarter',
  1211. 'schemas': None,
  1212. },
  1213. ]
  1214. extract_process = flexmock(stdout=flexmock())
  1215. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1216. 'resolve_credential',
  1217. ).replace_with(lambda value, config: value)
  1218. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1219. flexmock(module).should_receive('make_dump_path')
  1220. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1221. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1222. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1223. (
  1224. 'pg_restore',
  1225. '--no-password',
  1226. '--if-exists',
  1227. '--exit-on-error',
  1228. '--clean',
  1229. '--dbname',
  1230. 'foo',
  1231. '--harder',
  1232. ),
  1233. processes=[extract_process],
  1234. output_log_level=logging.DEBUG,
  1235. input_file=extract_process.stdout,
  1236. environment={'PGSSLMODE': 'disable'},
  1237. working_directory=None,
  1238. ).once()
  1239. flexmock(module).should_receive('execute_command').with_args(
  1240. (
  1241. 'psql',
  1242. '--no-password',
  1243. '--no-psqlrc',
  1244. '--quiet',
  1245. '--dbname',
  1246. 'foo',
  1247. '--smarter',
  1248. '--command',
  1249. 'ANALYZE',
  1250. ),
  1251. environment={'PGSSLMODE': 'disable'},
  1252. working_directory=None,
  1253. ).once()
  1254. module.restore_data_source_dump(
  1255. hook_config,
  1256. {},
  1257. data_source=hook_config[0],
  1258. dry_run=False,
  1259. extract_process=extract_process,
  1260. connection_params={
  1261. 'hostname': None,
  1262. 'port': None,
  1263. 'username': None,
  1264. 'password': None,
  1265. },
  1266. borgmatic_runtime_directory='/run/borgmatic',
  1267. )
  1268. def test_restore_data_source_dump_runs_psql_for_all_database_dump():
  1269. hook_config = [{'name': 'all', 'schemas': None}]
  1270. extract_process = flexmock(stdout=flexmock())
  1271. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1272. 'resolve_credential',
  1273. ).replace_with(lambda value, config: value)
  1274. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1275. flexmock(module).should_receive('make_dump_path')
  1276. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1277. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1278. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1279. (
  1280. 'psql',
  1281. '--no-password',
  1282. '--no-psqlrc',
  1283. ),
  1284. processes=[extract_process],
  1285. output_log_level=logging.DEBUG,
  1286. input_file=extract_process.stdout,
  1287. environment={'PGSSLMODE': 'disable'},
  1288. working_directory=None,
  1289. ).once()
  1290. flexmock(module).should_receive('execute_command').with_args(
  1291. ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
  1292. environment={'PGSSLMODE': 'disable'},
  1293. working_directory=None,
  1294. ).once()
  1295. module.restore_data_source_dump(
  1296. hook_config,
  1297. {},
  1298. data_source={'name': 'all'},
  1299. dry_run=False,
  1300. extract_process=extract_process,
  1301. connection_params={
  1302. 'hostname': None,
  1303. 'port': None,
  1304. 'username': None,
  1305. 'password': None,
  1306. },
  1307. borgmatic_runtime_directory='/run/borgmatic',
  1308. )
  1309. def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
  1310. hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
  1311. extract_process = flexmock(stdout=flexmock())
  1312. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1313. 'resolve_credential',
  1314. ).replace_with(lambda value, config: value)
  1315. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1316. flexmock(module).should_receive('make_dump_path')
  1317. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1318. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1319. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1320. ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
  1321. processes=[extract_process],
  1322. output_log_level=logging.DEBUG,
  1323. input_file=extract_process.stdout,
  1324. environment={'PGSSLMODE': 'disable'},
  1325. working_directory=None,
  1326. ).once()
  1327. flexmock(module).should_receive('execute_command').with_args(
  1328. (
  1329. 'psql',
  1330. '--no-password',
  1331. '--no-psqlrc',
  1332. '--quiet',
  1333. '--dbname',
  1334. 'foo',
  1335. '--command',
  1336. 'ANALYZE',
  1337. ),
  1338. environment={'PGSSLMODE': 'disable'},
  1339. working_directory=None,
  1340. ).once()
  1341. module.restore_data_source_dump(
  1342. hook_config,
  1343. {},
  1344. data_source=hook_config[0],
  1345. dry_run=False,
  1346. extract_process=extract_process,
  1347. connection_params={
  1348. 'hostname': None,
  1349. 'port': None,
  1350. 'username': None,
  1351. 'password': None,
  1352. },
  1353. borgmatic_runtime_directory='/run/borgmatic',
  1354. )
  1355. def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
  1356. hook_config = [
  1357. {
  1358. 'name': 'foo',
  1359. 'pg_restore_command': 'docker exec --workdir * mycontainer pg_restore',
  1360. 'psql_command': 'docker exec --workdir * mycontainer psql',
  1361. 'schemas': None,
  1362. },
  1363. ]
  1364. extract_process = flexmock(stdout=flexmock())
  1365. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1366. 'resolve_credential',
  1367. ).replace_with(lambda value, config: value)
  1368. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1369. flexmock(module).should_receive('make_dump_path')
  1370. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1371. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1372. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1373. (
  1374. 'docker',
  1375. 'exec',
  1376. '--workdir',
  1377. "'*'", # Should get shell escaped to prevent injection attacks.
  1378. 'mycontainer',
  1379. 'pg_restore',
  1380. '--no-password',
  1381. '--if-exists',
  1382. '--exit-on-error',
  1383. '--clean',
  1384. '--dbname',
  1385. 'foo',
  1386. ),
  1387. processes=[extract_process],
  1388. output_log_level=logging.DEBUG,
  1389. input_file=extract_process.stdout,
  1390. environment={'PGSSLMODE': 'disable'},
  1391. working_directory=None,
  1392. ).once()
  1393. flexmock(module).should_receive('execute_command').with_args(
  1394. (
  1395. 'docker',
  1396. 'exec',
  1397. '--workdir',
  1398. "'*'", # Should get shell escaped to prevent injection attacks.
  1399. 'mycontainer',
  1400. 'psql',
  1401. '--no-password',
  1402. '--no-psqlrc',
  1403. '--quiet',
  1404. '--dbname',
  1405. 'foo',
  1406. '--command',
  1407. 'ANALYZE',
  1408. ),
  1409. environment={'PGSSLMODE': 'disable'},
  1410. working_directory=None,
  1411. ).once()
  1412. module.restore_data_source_dump(
  1413. hook_config,
  1414. {},
  1415. data_source=hook_config[0],
  1416. dry_run=False,
  1417. extract_process=extract_process,
  1418. connection_params={
  1419. 'hostname': None,
  1420. 'port': None,
  1421. 'username': None,
  1422. 'password': None,
  1423. },
  1424. borgmatic_runtime_directory='/run/borgmatic',
  1425. )
  1426. def test_restore_data_source_dump_with_dry_run_skips_restore():
  1427. hook_config = [{'name': 'foo', 'schemas': None}]
  1428. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1429. 'resolve_credential',
  1430. ).replace_with(lambda value, config: value)
  1431. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1432. flexmock(module).should_receive('make_dump_path')
  1433. flexmock(module.dump).should_receive('make_data_source_dump_filename')
  1434. flexmock(module).should_receive('execute_command_with_processes').never()
  1435. module.restore_data_source_dump(
  1436. hook_config,
  1437. {},
  1438. data_source={'name': 'foo'},
  1439. dry_run=True,
  1440. extract_process=flexmock(),
  1441. connection_params={
  1442. 'hostname': None,
  1443. 'port': None,
  1444. 'username': None,
  1445. 'password': None,
  1446. },
  1447. borgmatic_runtime_directory='/run/borgmatic',
  1448. )
  1449. def test_restore_data_source_dump_without_extract_process_restores_from_disk():
  1450. hook_config = [{'name': 'foo', 'schemas': None}]
  1451. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1452. 'resolve_credential',
  1453. ).replace_with(lambda value, config: value)
  1454. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1455. flexmock(module).should_receive('make_dump_path')
  1456. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1457. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1458. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1459. (
  1460. 'pg_restore',
  1461. '--no-password',
  1462. '--if-exists',
  1463. '--exit-on-error',
  1464. '--clean',
  1465. '--dbname',
  1466. 'foo',
  1467. '/dump/path',
  1468. ),
  1469. processes=[],
  1470. output_log_level=logging.DEBUG,
  1471. input_file=None,
  1472. environment={'PGSSLMODE': 'disable'},
  1473. working_directory=None,
  1474. ).once()
  1475. flexmock(module).should_receive('execute_command').with_args(
  1476. (
  1477. 'psql',
  1478. '--no-password',
  1479. '--no-psqlrc',
  1480. '--quiet',
  1481. '--dbname',
  1482. 'foo',
  1483. '--command',
  1484. 'ANALYZE',
  1485. ),
  1486. environment={'PGSSLMODE': 'disable'},
  1487. working_directory=None,
  1488. ).once()
  1489. module.restore_data_source_dump(
  1490. hook_config,
  1491. {},
  1492. data_source={'name': 'foo'},
  1493. dry_run=False,
  1494. extract_process=None,
  1495. connection_params={
  1496. 'hostname': None,
  1497. 'port': None,
  1498. 'username': None,
  1499. 'password': None,
  1500. },
  1501. borgmatic_runtime_directory='/run/borgmatic',
  1502. )
  1503. def test_restore_data_source_dump_with_schemas_restores_schemas():
  1504. hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
  1505. flexmock(module.borgmatic.hooks.credential.parse).should_receive(
  1506. 'resolve_credential',
  1507. ).replace_with(lambda value, config: value)
  1508. flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'})
  1509. flexmock(module).should_receive('make_dump_path')
  1510. flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path')
  1511. flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
  1512. flexmock(module).should_receive('execute_command_with_processes').with_args(
  1513. (
  1514. 'pg_restore',
  1515. '--no-password',
  1516. '--if-exists',
  1517. '--exit-on-error',
  1518. '--clean',
  1519. '--dbname',
  1520. 'foo',
  1521. '/dump/path',
  1522. '--schema',
  1523. 'bar',
  1524. '--schema',
  1525. 'baz',
  1526. ),
  1527. processes=[],
  1528. output_log_level=logging.DEBUG,
  1529. input_file=None,
  1530. environment={'PGSSLMODE': 'disable'},
  1531. working_directory=None,
  1532. ).once()
  1533. flexmock(module).should_receive('execute_command').with_args(
  1534. (
  1535. 'psql',
  1536. '--no-password',
  1537. '--no-psqlrc',
  1538. '--quiet',
  1539. '--dbname',
  1540. 'foo',
  1541. '--command',
  1542. 'ANALYZE',
  1543. ),
  1544. environment={'PGSSLMODE': 'disable'},
  1545. working_directory=None,
  1546. ).once()
  1547. module.restore_data_source_dump(
  1548. hook_config,
  1549. {},
  1550. data_source=hook_config[0],
  1551. dry_run=False,
  1552. extract_process=None,
  1553. connection_params={
  1554. 'hostname': None,
  1555. 'port': None,
  1556. 'username': None,
  1557. 'password': None,
  1558. },
  1559. borgmatic_runtime_directory='/run/borgmatic',
  1560. )