test_postgresql.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862
  1. import logging
  2. import pytest
  3. from flexmock import flexmock
  4. from borgmatic.hooks import postgresql as module
  5. def test_database_names_to_dump_passes_through_individual_database_name():
  6. database = {'name': 'foo'}
  7. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  8. 'foo',
  9. )
  10. def test_database_names_to_dump_passes_through_individual_database_name_with_format():
  11. database = {'name': 'foo', 'format': 'custom'}
  12. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  13. 'foo',
  14. )
  15. def test_database_names_to_dump_passes_through_all_without_format():
  16. database = {'name': 'all'}
  17. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  18. 'all',
  19. )
  20. def test_database_names_to_dump_with_all_and_format_and_dry_run_bails():
  21. database = {'name': 'all', 'format': 'custom'}
  22. flexmock(module).should_receive('execute_command_and_capture_output').never()
  23. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=True) == ()
  24. def test_database_names_to_dump_with_all_and_format_lists_databases():
  25. database = {'name': 'all', 'format': 'custom'}
  26. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  27. 'foo,test,\nbar,test,"stuff and such"'
  28. )
  29. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  30. 'foo',
  31. 'bar',
  32. )
  33. def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
  34. database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
  35. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  36. (
  37. 'psql',
  38. '--list',
  39. '--no-password',
  40. '--no-psqlrc',
  41. '--csv',
  42. '--tuples-only',
  43. '--host',
  44. 'localhost',
  45. '--port',
  46. '1234',
  47. ),
  48. extra_environment=object,
  49. ).and_return('foo,test,\nbar,test,"stuff and such"')
  50. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  51. 'foo',
  52. 'bar',
  53. )
  54. def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
  55. database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
  56. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  57. (
  58. 'psql',
  59. '--list',
  60. '--no-password',
  61. '--no-psqlrc',
  62. '--csv',
  63. '--tuples-only',
  64. '--username',
  65. 'postgres',
  66. ),
  67. extra_environment=object,
  68. ).and_return('foo,test,\nbar,test,"stuff and such"')
  69. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  70. 'foo',
  71. 'bar',
  72. )
  73. def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
  74. database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
  75. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  76. ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
  77. extra_environment=object,
  78. ).and_return('foo,test,\nbar,test,"stuff and such"')
  79. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  80. 'foo',
  81. 'bar',
  82. )
  83. def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
  84. database = {'name': 'all', 'format': 'custom'}
  85. flexmock(module).should_receive('execute_command_and_capture_output').and_return(
  86. 'foo,test,\ntemplate0,test,blah'
  87. )
  88. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  89. 'foo',
  90. )
  91. def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
  92. database = {'name': 'all', 'format': 'custom', 'psql_command': 'docker exec mycontainer psql'}
  93. flexmock(module).should_receive('execute_command_and_capture_output').with_args(
  94. (
  95. 'docker',
  96. 'exec',
  97. 'mycontainer',
  98. 'psql',
  99. '--list',
  100. '--no-password',
  101. '--no-psqlrc',
  102. '--csv',
  103. '--tuples-only',
  104. ),
  105. extra_environment=object,
  106. ).and_return('foo,text').once()
  107. assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == (
  108. 'foo',
  109. )
  110. def test_dump_databases_runs_pg_dump_for_each_database():
  111. databases = [{'name': 'foo'}, {'name': 'bar'}]
  112. processes = [flexmock(), flexmock()]
  113. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  114. flexmock(module).should_receive('make_dump_path').and_return('')
  115. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  116. ('bar',)
  117. )
  118. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  119. 'databases/localhost/foo'
  120. ).and_return('databases/localhost/bar')
  121. flexmock(module.os.path).should_receive('exists').and_return(False)
  122. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  123. for name, process in zip(('foo', 'bar'), processes):
  124. flexmock(module).should_receive('execute_command').with_args(
  125. (
  126. 'pg_dump',
  127. '--no-password',
  128. '--clean',
  129. '--if-exists',
  130. '--format',
  131. 'custom',
  132. name,
  133. '>',
  134. f'databases/localhost/{name}',
  135. ),
  136. shell=True,
  137. extra_environment={'PGSSLMODE': 'disable'},
  138. run_to_completion=False,
  139. ).and_return(process).once()
  140. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
  141. def test_dump_databases_raises_when_no_database_names_to_dump():
  142. databases = [{'name': 'foo'}, {'name': 'bar'}]
  143. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  144. flexmock(module).should_receive('make_dump_path').and_return('')
  145. flexmock(module).should_receive('database_names_to_dump').and_return(())
  146. with pytest.raises(ValueError):
  147. module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
  148. def test_dump_databases_does_not_raise_when_no_database_names_to_dump():
  149. databases = [{'name': 'foo'}, {'name': 'bar'}]
  150. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  151. flexmock(module).should_receive('make_dump_path').and_return('')
  152. flexmock(module).should_receive('database_names_to_dump').and_return(())
  153. module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
  154. def test_dump_databases_with_duplicate_dump_skips_pg_dump():
  155. databases = [{'name': 'foo'}, {'name': 'bar'}]
  156. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  157. flexmock(module).should_receive('make_dump_path').and_return('')
  158. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  159. ('bar',)
  160. )
  161. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  162. 'databases/localhost/foo'
  163. ).and_return('databases/localhost/bar')
  164. flexmock(module.os.path).should_receive('exists').and_return(True)
  165. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  166. flexmock(module).should_receive('execute_command').never()
  167. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == []
  168. def test_dump_databases_with_dry_run_skips_pg_dump():
  169. databases = [{'name': 'foo'}, {'name': 'bar'}]
  170. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  171. flexmock(module).should_receive('make_dump_path').and_return('')
  172. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
  173. ('bar',)
  174. )
  175. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  176. 'databases/localhost/foo'
  177. ).and_return('databases/localhost/bar')
  178. flexmock(module.os.path).should_receive('exists').and_return(False)
  179. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  180. flexmock(module).should_receive('execute_command').never()
  181. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
  182. def test_dump_databases_runs_pg_dump_with_hostname_and_port():
  183. databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
  184. process = flexmock()
  185. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  186. flexmock(module).should_receive('make_dump_path').and_return('')
  187. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  188. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  189. 'databases/database.example.org/foo'
  190. )
  191. flexmock(module.os.path).should_receive('exists').and_return(False)
  192. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  193. flexmock(module).should_receive('execute_command').with_args(
  194. (
  195. 'pg_dump',
  196. '--no-password',
  197. '--clean',
  198. '--if-exists',
  199. '--host',
  200. 'database.example.org',
  201. '--port',
  202. '5433',
  203. '--format',
  204. 'custom',
  205. 'foo',
  206. '>',
  207. 'databases/database.example.org/foo',
  208. ),
  209. shell=True,
  210. extra_environment={'PGSSLMODE': 'disable'},
  211. run_to_completion=False,
  212. ).and_return(process).once()
  213. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
  214. def test_dump_databases_runs_pg_dump_with_username_and_password():
  215. databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
  216. process = flexmock()
  217. flexmock(module).should_receive('make_extra_environment').and_return(
  218. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
  219. )
  220. flexmock(module).should_receive('make_dump_path').and_return('')
  221. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  222. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  223. 'databases/localhost/foo'
  224. )
  225. flexmock(module.os.path).should_receive('exists').and_return(False)
  226. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  227. flexmock(module).should_receive('execute_command').with_args(
  228. (
  229. 'pg_dump',
  230. '--no-password',
  231. '--clean',
  232. '--if-exists',
  233. '--username',
  234. 'postgres',
  235. '--format',
  236. 'custom',
  237. 'foo',
  238. '>',
  239. 'databases/localhost/foo',
  240. ),
  241. shell=True,
  242. extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  243. run_to_completion=False,
  244. ).and_return(process).once()
  245. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
  246. def test_make_extra_environment_maps_options_to_environment():
  247. database = {
  248. 'name': 'foo',
  249. 'password': 'pass',
  250. 'ssl_mode': 'require',
  251. 'ssl_cert': 'cert.crt',
  252. 'ssl_key': 'key.key',
  253. 'ssl_root_cert': 'root.crt',
  254. 'ssl_crl': 'crl.crl',
  255. }
  256. expected = {
  257. 'PGPASSWORD': 'pass',
  258. 'PGSSLMODE': 'require',
  259. 'PGSSLCERT': 'cert.crt',
  260. 'PGSSLKEY': 'key.key',
  261. 'PGSSLROOTCERT': 'root.crt',
  262. 'PGSSLCRL': 'crl.crl',
  263. }
  264. extra_env = module.make_extra_environment(database)
  265. assert extra_env == expected
  266. def test_dump_databases_runs_pg_dump_with_directory_format():
  267. databases = [{'name': 'foo', 'format': 'directory'}]
  268. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  269. flexmock(module).should_receive('make_dump_path').and_return('')
  270. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  271. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  272. 'databases/localhost/foo'
  273. )
  274. flexmock(module.os.path).should_receive('exists').and_return(False)
  275. flexmock(module.dump).should_receive('create_parent_directory_for_dump')
  276. flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
  277. flexmock(module).should_receive('execute_command').with_args(
  278. (
  279. 'pg_dump',
  280. '--no-password',
  281. '--clean',
  282. '--if-exists',
  283. '--format',
  284. 'directory',
  285. '--file',
  286. 'databases/localhost/foo',
  287. 'foo',
  288. ),
  289. shell=True,
  290. extra_environment={'PGSSLMODE': 'disable'},
  291. ).and_return(flexmock()).once()
  292. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == []
  293. def test_dump_databases_runs_pg_dump_with_options():
  294. databases = [{'name': 'foo', 'options': '--stuff=such'}]
  295. process = flexmock()
  296. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  297. flexmock(module).should_receive('make_dump_path').and_return('')
  298. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  299. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  300. 'databases/localhost/foo'
  301. )
  302. flexmock(module.os.path).should_receive('exists').and_return(False)
  303. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  304. flexmock(module).should_receive('execute_command').with_args(
  305. (
  306. 'pg_dump',
  307. '--no-password',
  308. '--clean',
  309. '--if-exists',
  310. '--format',
  311. 'custom',
  312. '--stuff=such',
  313. 'foo',
  314. '>',
  315. 'databases/localhost/foo',
  316. ),
  317. shell=True,
  318. extra_environment={'PGSSLMODE': 'disable'},
  319. run_to_completion=False,
  320. ).and_return(process).once()
  321. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
  322. def test_dump_databases_runs_pg_dumpall_for_all_databases():
  323. databases = [{'name': 'all'}]
  324. process = flexmock()
  325. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  326. flexmock(module).should_receive('make_dump_path').and_return('')
  327. flexmock(module).should_receive('database_names_to_dump').and_return(('all',))
  328. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  329. 'databases/localhost/all'
  330. )
  331. flexmock(module.os.path).should_receive('exists').and_return(False)
  332. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  333. flexmock(module).should_receive('execute_command').with_args(
  334. ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
  335. shell=True,
  336. extra_environment={'PGSSLMODE': 'disable'},
  337. run_to_completion=False,
  338. ).and_return(process).once()
  339. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
  340. def test_dump_databases_runs_non_default_pg_dump():
  341. databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump'}]
  342. process = flexmock()
  343. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  344. flexmock(module).should_receive('make_dump_path').and_return('')
  345. flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
  346. flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
  347. 'databases/localhost/foo'
  348. )
  349. flexmock(module.os.path).should_receive('exists').and_return(False)
  350. flexmock(module.dump).should_receive('create_named_pipe_for_dump')
  351. flexmock(module).should_receive('execute_command').with_args(
  352. (
  353. 'special_pg_dump',
  354. '--no-password',
  355. '--clean',
  356. '--if-exists',
  357. '--format',
  358. 'custom',
  359. 'foo',
  360. '>',
  361. 'databases/localhost/foo',
  362. ),
  363. shell=True,
  364. extra_environment={'PGSSLMODE': 'disable'},
  365. run_to_completion=False,
  366. ).and_return(process).once()
  367. assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
  368. def test_restore_database_dump_runs_pg_restore():
  369. database_config = [{'name': 'foo', 'schemas': None}]
  370. extract_process = flexmock(stdout=flexmock())
  371. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  372. flexmock(module).should_receive('make_dump_path')
  373. flexmock(module.dump).should_receive('make_database_dump_filename')
  374. flexmock(module).should_receive('execute_command_with_processes').with_args(
  375. (
  376. 'pg_restore',
  377. '--no-password',
  378. '--if-exists',
  379. '--exit-on-error',
  380. '--clean',
  381. '--dbname',
  382. 'foo',
  383. ),
  384. processes=[extract_process],
  385. output_log_level=logging.DEBUG,
  386. input_file=extract_process.stdout,
  387. extra_environment={'PGSSLMODE': 'disable'},
  388. ).once()
  389. flexmock(module).should_receive('execute_command').with_args(
  390. (
  391. 'psql',
  392. '--no-password',
  393. '--no-psqlrc',
  394. '--quiet',
  395. '--dbname',
  396. 'foo',
  397. '--command',
  398. 'ANALYZE',
  399. ),
  400. extra_environment={'PGSSLMODE': 'disable'},
  401. ).once()
  402. module.restore_database_dump(
  403. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  404. )
  405. def test_restore_database_dump_errors_on_multiple_database_config():
  406. database_config = [{'name': 'foo'}, {'name': 'bar'}]
  407. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  408. flexmock(module).should_receive('make_dump_path')
  409. flexmock(module.dump).should_receive('make_database_dump_filename')
  410. flexmock(module).should_receive('execute_command_with_processes').never()
  411. flexmock(module).should_receive('execute_command').never()
  412. with pytest.raises(ValueError):
  413. module.restore_database_dump(
  414. database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
  415. )
  416. def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
  417. database_config = [
  418. {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None}
  419. ]
  420. extract_process = flexmock(stdout=flexmock())
  421. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  422. flexmock(module).should_receive('make_dump_path')
  423. flexmock(module.dump).should_receive('make_database_dump_filename')
  424. flexmock(module).should_receive('execute_command_with_processes').with_args(
  425. (
  426. 'pg_restore',
  427. '--no-password',
  428. '--if-exists',
  429. '--exit-on-error',
  430. '--clean',
  431. '--dbname',
  432. 'foo',
  433. '--host',
  434. 'database.example.org',
  435. '--port',
  436. '5433',
  437. ),
  438. processes=[extract_process],
  439. output_log_level=logging.DEBUG,
  440. input_file=extract_process.stdout,
  441. extra_environment={'PGSSLMODE': 'disable'},
  442. ).once()
  443. flexmock(module).should_receive('execute_command').with_args(
  444. (
  445. 'psql',
  446. '--no-password',
  447. '--no-psqlrc',
  448. '--quiet',
  449. '--host',
  450. 'database.example.org',
  451. '--port',
  452. '5433',
  453. '--dbname',
  454. 'foo',
  455. '--command',
  456. 'ANALYZE',
  457. ),
  458. extra_environment={'PGSSLMODE': 'disable'},
  459. ).once()
  460. module.restore_database_dump(
  461. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  462. )
  463. def test_restore_database_dump_runs_pg_restore_with_username_and_password():
  464. database_config = [
  465. {'name': 'foo', 'username': 'postgres', 'password': 'trustsome1', 'schemas': None}
  466. ]
  467. extract_process = flexmock(stdout=flexmock())
  468. flexmock(module).should_receive('make_extra_environment').and_return(
  469. {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
  470. )
  471. flexmock(module).should_receive('make_dump_path')
  472. flexmock(module.dump).should_receive('make_database_dump_filename')
  473. flexmock(module).should_receive('execute_command_with_processes').with_args(
  474. (
  475. 'pg_restore',
  476. '--no-password',
  477. '--if-exists',
  478. '--exit-on-error',
  479. '--clean',
  480. '--dbname',
  481. 'foo',
  482. '--username',
  483. 'postgres',
  484. ),
  485. processes=[extract_process],
  486. output_log_level=logging.DEBUG,
  487. input_file=extract_process.stdout,
  488. extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  489. ).once()
  490. flexmock(module).should_receive('execute_command').with_args(
  491. (
  492. 'psql',
  493. '--no-password',
  494. '--no-psqlrc',
  495. '--quiet',
  496. '--username',
  497. 'postgres',
  498. '--dbname',
  499. 'foo',
  500. '--command',
  501. 'ANALYZE',
  502. ),
  503. extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
  504. ).once()
  505. module.restore_database_dump(
  506. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  507. )
  508. def test_restore_database_dump_runs_pg_restore_with_options():
  509. database_config = [
  510. {
  511. 'name': 'foo',
  512. 'restore_options': '--harder',
  513. 'analyze_options': '--smarter',
  514. 'schemas': None,
  515. }
  516. ]
  517. extract_process = flexmock(stdout=flexmock())
  518. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  519. flexmock(module).should_receive('make_dump_path')
  520. flexmock(module.dump).should_receive('make_database_dump_filename')
  521. flexmock(module).should_receive('execute_command_with_processes').with_args(
  522. (
  523. 'pg_restore',
  524. '--no-password',
  525. '--if-exists',
  526. '--exit-on-error',
  527. '--clean',
  528. '--dbname',
  529. 'foo',
  530. '--harder',
  531. ),
  532. processes=[extract_process],
  533. output_log_level=logging.DEBUG,
  534. input_file=extract_process.stdout,
  535. extra_environment={'PGSSLMODE': 'disable'},
  536. ).once()
  537. flexmock(module).should_receive('execute_command').with_args(
  538. (
  539. 'psql',
  540. '--no-password',
  541. '--no-psqlrc',
  542. '--quiet',
  543. '--dbname',
  544. 'foo',
  545. '--smarter',
  546. '--command',
  547. 'ANALYZE',
  548. ),
  549. extra_environment={'PGSSLMODE': 'disable'},
  550. ).once()
  551. module.restore_database_dump(
  552. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  553. )
  554. def test_restore_database_dump_runs_psql_for_all_database_dump():
  555. database_config = [{'name': 'all', 'schemas': None}]
  556. extract_process = flexmock(stdout=flexmock())
  557. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  558. flexmock(module).should_receive('make_dump_path')
  559. flexmock(module.dump).should_receive('make_database_dump_filename')
  560. flexmock(module).should_receive('execute_command_with_processes').with_args(
  561. (
  562. 'psql',
  563. '--no-password',
  564. '--no-psqlrc',
  565. ),
  566. processes=[extract_process],
  567. output_log_level=logging.DEBUG,
  568. input_file=extract_process.stdout,
  569. extra_environment={'PGSSLMODE': 'disable'},
  570. ).once()
  571. flexmock(module).should_receive('execute_command').with_args(
  572. ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
  573. extra_environment={'PGSSLMODE': 'disable'},
  574. ).once()
  575. module.restore_database_dump(
  576. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  577. )
  578. def test_restore_database_dump_runs_psql_for_plain_database_dump():
  579. database_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}]
  580. extract_process = flexmock(stdout=flexmock())
  581. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  582. flexmock(module).should_receive('make_dump_path')
  583. flexmock(module.dump).should_receive('make_database_dump_filename')
  584. flexmock(module).should_receive('execute_command_with_processes').with_args(
  585. ('psql', '--no-password', '--no-psqlrc', '--dbname', 'foo'),
  586. processes=[extract_process],
  587. output_log_level=logging.DEBUG,
  588. input_file=extract_process.stdout,
  589. extra_environment={'PGSSLMODE': 'disable'},
  590. ).once()
  591. flexmock(module).should_receive('execute_command').with_args(
  592. (
  593. 'psql',
  594. '--no-password',
  595. '--no-psqlrc',
  596. '--quiet',
  597. '--dbname',
  598. 'foo',
  599. '--command',
  600. 'ANALYZE',
  601. ),
  602. extra_environment={'PGSSLMODE': 'disable'},
  603. ).once()
  604. module.restore_database_dump(
  605. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  606. )
  607. def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
  608. database_config = [
  609. {
  610. 'name': 'foo',
  611. 'pg_restore_command': 'docker exec mycontainer pg_restore',
  612. 'psql_command': 'docker exec mycontainer psql',
  613. 'schemas': None,
  614. }
  615. ]
  616. extract_process = flexmock(stdout=flexmock())
  617. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  618. flexmock(module).should_receive('make_dump_path')
  619. flexmock(module.dump).should_receive('make_database_dump_filename')
  620. flexmock(module).should_receive('execute_command_with_processes').with_args(
  621. (
  622. 'docker',
  623. 'exec',
  624. 'mycontainer',
  625. 'pg_restore',
  626. '--no-password',
  627. '--if-exists',
  628. '--exit-on-error',
  629. '--clean',
  630. '--dbname',
  631. 'foo',
  632. ),
  633. processes=[extract_process],
  634. output_log_level=logging.DEBUG,
  635. input_file=extract_process.stdout,
  636. extra_environment={'PGSSLMODE': 'disable'},
  637. ).once()
  638. flexmock(module).should_receive('execute_command').with_args(
  639. (
  640. 'docker',
  641. 'exec',
  642. 'mycontainer',
  643. 'psql',
  644. '--no-password',
  645. '--no-psqlrc',
  646. '--quiet',
  647. '--dbname',
  648. 'foo',
  649. '--command',
  650. 'ANALYZE',
  651. ),
  652. extra_environment={'PGSSLMODE': 'disable'},
  653. ).once()
  654. module.restore_database_dump(
  655. database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
  656. )
  657. def test_restore_database_dump_with_dry_run_skips_restore():
  658. database_config = [{'name': 'foo', 'schemas': None}]
  659. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  660. flexmock(module).should_receive('make_dump_path')
  661. flexmock(module.dump).should_receive('make_database_dump_filename')
  662. flexmock(module).should_receive('execute_command_with_processes').never()
  663. module.restore_database_dump(
  664. database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
  665. )
  666. def test_restore_database_dump_without_extract_process_restores_from_disk():
  667. database_config = [{'name': 'foo', 'schemas': None}]
  668. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  669. flexmock(module).should_receive('make_dump_path')
  670. flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
  671. flexmock(module).should_receive('execute_command_with_processes').with_args(
  672. (
  673. 'pg_restore',
  674. '--no-password',
  675. '--if-exists',
  676. '--exit-on-error',
  677. '--clean',
  678. '--dbname',
  679. 'foo',
  680. '/dump/path',
  681. ),
  682. processes=[],
  683. output_log_level=logging.DEBUG,
  684. input_file=None,
  685. extra_environment={'PGSSLMODE': 'disable'},
  686. ).once()
  687. flexmock(module).should_receive('execute_command').with_args(
  688. (
  689. 'psql',
  690. '--no-password',
  691. '--no-psqlrc',
  692. '--quiet',
  693. '--dbname',
  694. 'foo',
  695. '--command',
  696. 'ANALYZE',
  697. ),
  698. extra_environment={'PGSSLMODE': 'disable'},
  699. ).once()
  700. module.restore_database_dump(
  701. database_config, 'test.yaml', {}, dry_run=False, extract_process=None
  702. )
  703. def test_restore_database_dump_with_schemas_restores_schemas():
  704. database_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}]
  705. flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
  706. flexmock(module).should_receive('make_dump_path')
  707. flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
  708. flexmock(module).should_receive('execute_command_with_processes').with_args(
  709. (
  710. 'pg_restore',
  711. '--no-password',
  712. '--if-exists',
  713. '--exit-on-error',
  714. '--clean',
  715. '--dbname',
  716. 'foo',
  717. '/dump/path',
  718. '--schema',
  719. 'bar',
  720. '--schema',
  721. 'baz',
  722. ),
  723. processes=[],
  724. output_log_level=logging.DEBUG,
  725. input_file=None,
  726. extra_environment={'PGSSLMODE': 'disable'},
  727. ).once()
  728. flexmock(module).should_receive('execute_command').with_args(
  729. (
  730. 'psql',
  731. '--no-password',
  732. '--no-psqlrc',
  733. '--quiet',
  734. '--dbname',
  735. 'foo',
  736. '--command',
  737. 'ANALYZE',
  738. ),
  739. extra_environment={'PGSSLMODE': 'disable'},
  740. ).once()
  741. module.restore_database_dump(
  742. database_config, 'test.yaml', {}, dry_run=False, extract_process=None
  743. )