Fix tests for database hooks.

NEWS

@@ -4,7 +4,7 @@
    use.
  * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
    with orphaned files that need recovery.
- * #1136: For all database hooks, record metadata on the dumps contained within an archive.
+ * #1136: For all database hooks, record metadata about the dumps contained within an archive.
  * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
  * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
    replacing Flake8, Black, isort, etc.

borgmatic/hooks/data_source/mariadb.py

@@ -294,14 +294,13 @@ def dump_data_sources(
 
             raise ValueError('Cannot find any MariaDB databases to dump.')
 
-
         if database['name'] == 'all' and database.get('format'):
             for database_name in dump_database_names:
                 dumps_metadata.append(
                     borgmatic.actions.restore.Dump(
                         'mariadb_databases',
                         database_name,
-                        database.get('hostname'),
+                        database.get('hostname', 'localhost'),
                         database.get('port'),
                     )
                 )
@@ -325,7 +324,7 @@ def dump_data_sources(
                 borgmatic.actions.restore.Dump(
                     'mariadb_databases',
                     database['name'],
-                    database.get('hostname'),
+                    database.get('hostname', 'localhost'),
                     database.get('port'),
                 )
             )

borgmatic/hooks/data_source/mongodb.py

@@ -61,7 +61,7 @@ def dump_data_sources(
             borgmatic.actions.restore.Dump(
                 'mongodb_databases',
                 name,
-                database.get('hostname'),
+                database.get('hostname', 'localhost'),
                 database.get('port'),
             )
         )

borgmatic/hooks/data_source/mysql.py

@@ -225,14 +225,13 @@ def dump_data_sources(
 
             raise ValueError('Cannot find any MySQL databases to dump.')
 
-
         if database['name'] == 'all' and database.get('format'):
             for database_name in dump_database_names:
                 dumps_metadata.append(
                     borgmatic.actions.restore.Dump(
                         'mysql_databases',
                         database_name,
-                        database.get('hostname'),
+                        database.get('hostname', 'localhost'),
                         database.get('port'),
                     )
                 )
@@ -256,7 +255,7 @@ def dump_data_sources(
                 borgmatic.actions.restore.Dump(
                     'mysql_databases',
                     database['name'],
-                    database.get('hostname'),
+                    database.get('hostname', 'localhost'),
                     database.get('port'),
                 )
             )

borgmatic/hooks/data_source/postgresql.py

@@ -170,7 +170,7 @@ def dump_data_sources(
                 borgmatic.actions.restore.Dump(
                     'postgresql_databases',
                     database_name,
-                    database.get('hostname'),
+                    database.get('hostname', 'localhost'),
                     database.get('port'),
                 )
             )

tests/unit/hooks/data_source/test_mariadb.py

@@ -349,6 +349,15 @@ def test_dump_data_sources_dumps_each_database():
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'),
+        ]
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -390,6 +399,13 @@ def test_dump_data_sources_dumps_with_password():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -434,6 +450,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -465,6 +488,13 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'all'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -499,6 +529,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mariadb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mariadb_databases', 'bar'),
+        ]
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,

tests/unit/hooks/data_source/test_mongodb.py

@@ -39,6 +39,15 @@ def test_dump_data_sources_runs_mongodump_for_each_database():
             run_to_completion=False,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'bar'),
+        ]
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -60,6 +69,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
     ).and_return('databases/localhost/bar')
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(
@@ -75,7 +85,7 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
 
 
 def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 27018}]
     process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return(
@@ -89,7 +99,7 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
             '--host',
             'database.example.org',
             '--port',
-            '5433',
+            '27018',
             '--db',
             'foo',
             '--archive',
@@ -99,6 +109,13 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo', 'database.example.org', 27018),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -150,6 +167,13 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -174,6 +198,13 @@ def test_dump_data_sources_runs_mongodump_with_directory_format():
         ('mongodump', '--out', 'databases/localhost/foo', '--db', 'foo'),
         shell=True,
     ).and_return(flexmock()).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -210,6 +241,13 @@ def test_dump_data_sources_runs_mongodump_with_options():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -235,6 +273,13 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'all'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -308,7 +353,7 @@ def test_restore_data_source_dump_runs_mongorestore():
 
 def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
     hook_config = [
-        {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None},
+        {'name': 'foo', 'hostname': 'database.example.org', 'port': 27018, 'schemas': None},
     ]
     extract_process = flexmock(stdout=flexmock())
 
@@ -325,7 +370,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
             '--host',
             'database.example.org',
             '--port',
-            '5433',
+            '27018',
         ],
         processes=[extract_process],
         output_log_level=logging.DEBUG,
@@ -708,6 +753,13 @@ def test_dump_data_sources_uses_custom_mongodump_command():
         shell=True,
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mongodb_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mongodb_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,

tests/unit/hooks/data_source/test_mysql.py

@@ -232,6 +232,15 @@ def test_dump_data_sources_dumps_each_database():
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'),
+        ]
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,
@@ -273,6 +282,13 @@ def test_dump_data_sources_dumps_with_password():
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -317,6 +333,13 @@ def test_dump_data_sources_dumps_with_environment_password_transport_passes_pass
         dry_run=object,
         dry_run_label=object,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         [database],
@@ -349,6 +372,14 @@ def test_dump_data_sources_dumps_all_databases_at_once():
         dry_run_label=object,
     ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'all'),
+        ]
+    ).once()
+
     assert module.dump_data_sources(
         databases,
         {},
@@ -382,6 +413,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
             dry_run_label=object,
         ).and_return(process).once()
 
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'mysql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('mysql_databases', 'bar'),
+        ]
+    ).once()
+
     assert (
         module.dump_data_sources(
             databases,

tests/unit/hooks/data_source/test_postgresql.py

@@ -273,6 +273,14 @@ def test_dump_data_sources_runs_pg_dump_for_each_database():
             environment={'PGSSLMODE': 'disable'},
             run_to_completion=False,
         ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -336,6 +344,14 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'bar'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -366,6 +382,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump():
     ).replace_with(lambda value, config: value)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(
@@ -415,6 +432,13 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo', 'database.example.org', 5433),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -461,6 +485,13 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password():
         environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -507,6 +538,13 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped():
         environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -548,6 +586,13 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format():
         shell=True,
         environment={'PGSSLMODE': 'disable'},
     ).and_return(flexmock()).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -595,6 +640,13 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(processes[0]).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -642,6 +694,13 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(processes[0]).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -688,6 +747,13 @@ def test_dump_data_sources_runs_pg_dump_with_options():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -720,6 +786,13 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'all'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,
@@ -764,6 +837,13 @@ def test_dump_data_sources_runs_non_default_pg_dump():
         environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'postgresql_databases',
+        [
+            module.borgmatic.actions.restore.Dump('postgresql_databases', 'foo'),
+        ]
+    ).once()
 
     assert module.dump_data_sources(
         databases,

tests/unit/hooks/data_source/test_sqlite.py

@@ -26,6 +26,13 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists():
     flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -56,6 +63,14 @@ def test_dump_data_sources_dumps_each_database():
     flexmock(module).should_receive('execute_command').and_return(processes[0]).and_return(
         processes[1],
     )
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database2'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -93,6 +108,13 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped():
         shell=True,
         run_to_completion=False,
     ).and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -135,6 +157,13 @@ def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_ge
         shell=True,
         run_to_completion=False,
     ).and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -163,6 +192,13 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
     flexmock(module).should_receive('execute_command').and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'database1'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -193,6 +229,13 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
     flexmock(module).should_receive('execute_command').and_return(processes[0])
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').with_args(
+        '/run/borgmatic',
+        'sqlite_databases',
+        [
+            module.borgmatic.actions.restore.Dump('sqlite_databases', 'all'),
+        ]
+    ).once()
 
     assert (
         module.dump_data_sources(
@@ -217,6 +260,7 @@ def test_dump_data_sources_does_not_dump_if_dry_run():
     flexmock(module.os.path).should_receive('exists').and_return(False)
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
+    flexmock(module.dump).should_receive('write_data_source_dumps_metadata').never()
 
     assert (
         module.dump_data_sources(