Merge branch 'main' into fix-borg-2-latest-archive

NEWS

@@ -1,5 +1,9 @@
 2.0.8.dev0
  * #1114: Document systemd configuration changes for the ZFS filesystem hook.
+ * #1116: Add dumping of database containers via their container names, handy for backing up
+   database containers from the host. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
+ * #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
  * #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
    use.
  * #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always

borgmatic/actions/restore.py

@@ -22,8 +22,8 @@ UNSPECIFIED = object()
 
 Dump = collections.namedtuple(
     'Dump',
-    ('hook_name', 'data_source_name', 'hostname', 'port'),
-    defaults=('localhost', None),
+    ('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
+    defaults=('localhost', None, None, None),
 )
 
 
@@ -33,7 +33,13 @@ def dumps_match(first, second, default_port=None):
     indicates that the field should match any value. If a default port is given, then consider any
     dump having that port to match with a dump having a None port.
     '''
-    for field_name in first._fields:
+    # label kinda counts as an unique id, if they match ignore host/container/port
+    if first.label not in {None, UNSPECIFIED} and first.label == second.label:
+        field_list = ('hook_name', 'data_source_name')
+    else:
+        field_list = Dump._fields
+
+    for field_name in field_list:
         first_value = getattr(first, field_name)
         second_value = getattr(second, field_name)
 
@@ -57,14 +63,17 @@ def render_dump_metadata(dump):
     '''
     Given a Dump instance, make a display string describing it for use in log messages.
     '''
+    label = dump.label or UNSPECIFIED
     name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
-    hostname = dump.hostname or UNSPECIFIED
+    host = dump.container or dump.hostname or UNSPECIFIED
     port = None if dump.port is UNSPECIFIED else dump.port
 
-    if port:
-        metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
+    if label is not UNSPECIFIED:
+        metadata = f'{name}@{label}'
+    elif port:
+        metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
     else:
-        metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
+        metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
 
     if dump.hook_name not in {None, UNSPECIFIED}:
         return f'{metadata} ({dump.hook_name})'
@@ -101,6 +110,8 @@ def get_configured_data_source(config, restore_dump):
                 hook_data_source.get('name'),
                 hook_data_source.get('hostname', 'localhost'),
                 hook_data_source.get('port'),
+                hook_data_source.get('label') or UNSPECIFIED,
+                hook_data_source.get('container'),
             ),
             restore_dump,
             default_port,
@@ -172,7 +183,14 @@ def restore_single_dump(
     that data source from the archive.
     '''
     dump_metadata = render_dump_metadata(
-        Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port')),
+        Dump(
+            hook_name,
+            data_source['name'],
+            data_source.get('hostname'),
+            data_source.get('port'),
+            data_source.get('label') or UNSPECIFIED,
+            data_source.get('container'),
+        ),
     )
 
     logger.info(f'Restoring data source {dump_metadata}')
@@ -408,6 +426,8 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
                 data_source_name=name,
                 hostname=restore_arguments.original_hostname or UNSPECIFIED,
                 port=restore_arguments.original_port,
+                label=restore_arguments.original_label or UNSPECIFIED,
+                container=restore_arguments.original_container or UNSPECIFIED,
             )
             for name in restore_arguments.data_sources or (UNSPECIFIED,)
         }
@@ -415,12 +435,16 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
         or restore_arguments.data_sources
         or restore_arguments.original_hostname
         or restore_arguments.original_port
+        or restore_arguments.original_label
+        or restore_arguments.original_container
         else {
             Dump(
                 hook_name=UNSPECIFIED,
                 data_source_name='all',
                 hostname=UNSPECIFIED,
                 port=UNSPECIFIED,
+                label=UNSPECIFIED,
+                container=UNSPECIFIED,
             ),
         }
     )
@@ -541,6 +565,7 @@ def run_restore(
 
         dumps_actually_restored = set()
         connection_params = {
+            'container': restore_arguments.container,
             'hostname': restore_arguments.hostname,
             'port': restore_arguments.port,
             'username': restore_arguments.username,
@@ -554,13 +579,19 @@ def run_restore(
                 config,
                 restore_dump,
             )
-
             # For a dump that wasn't found via an exact match in the configuration, try to fallback
             # to an "all" data source.
             if not found_data_source:
                 found_data_source = get_configured_data_source(
                     config,
-                    Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
+                    Dump(
+                        restore_dump.hook_name,
+                        'all',
+                        restore_dump.hostname,
+                        restore_dump.port,
+                        restore_dump.label,
+                        restore_dump.container,
+                    ),
                 )
 
                 if not found_data_source:

borgmatic/commands/arguments.py

@@ -1475,6 +1475,10 @@ def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
         '--port',
         help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--container',
+        help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration',
+    )
     restore_group.add_argument(
         '--username',
         help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
@@ -1487,10 +1491,18 @@ def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
         '--restore-path',
         help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
     )
+    restore_group.add_argument(
+        '--original-label',
+        help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-hostname',
         help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
     )
+    restore_group.add_argument(
+        '--original-container',
+        help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
+    )
     restore_group.add_argument(
         '--original-port',
         type=int,

borgmatic/config/arguments.py

@@ -146,9 +146,9 @@ def prepare_arguments_for_config(global_arguments, schema):
         keys = tuple(argument_name.split('.'))
         option_type = type_for_option(schema, keys)
 
-        # The argument doesn't correspond to any option in the schema, so ignore it. It's
-        # probably a flag that borgmatic has on the command-line but not in configuration.
-        if option_type is None:
+        # The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
+        # It's probably a flag that borgmatic has on the command-line but not in configuration.
+        if option_type in {'object', None}:
             continue
 
         prepared_values.append(

borgmatic/config/schema.yaml

@@ -1323,6 +1323,23 @@ properties:
                         implicitly enables read_special (see above) to support
                         dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1524,6 +1541,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1689,6 +1723,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |
@@ -1862,6 +1913,11 @@ properties:
                         read_special (see above) to support dump and restore
                         streaming.
                     example: /var/lib/sqlite/users.db
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
                 restore_path:
                     type: string
                     description: |
@@ -1910,6 +1966,23 @@ properties:
                         database hook implicitly enables read_special (see
                         above) to support dump and restore streaming.
                     example: users
+                label:
+                    type: string
+                    description: |
+                        Label to identify the database dump in the backup.
+                    example: my_backup_label
+                container:
+                    type: string
+                    description: |
+                        Container name/id to connect to. When specified the
+                        hostname is ignored. Requires docker/podman CLI.
+                    example: debian_stable
+                restore_container:
+                    type: string
+                    description: |
+                        Container name/id to restore to. Defaults to the
+                        "container" option.
+                    example: restore_container
                 hostname:
                     type: string
                     description: |

borgmatic/hooks/data_source/config.py

@@ -0,0 +1,104 @@
+import json
+import logging
+import shutil
+import subprocess
+
+from borgmatic.execute import execute_command_and_capture_output
+
+IS_A_HOOK = False
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_database_option(option, data_source, connection_params=None, restore=False):
+    '''
+    Resolves a database option from the given data source configuration dict and
+    connection parameters dict. If restore is set to True it will consider the
+    `restore_<option>` instead.
+
+    Returns the resolved option or None. Can raise a ValueError if the hostname lookup
+    results in a container IP check.
+    '''
+    # Special case `hostname` since it overlaps with `container`
+    if option == 'hostname':
+        return get_hostname_from_config(data_source, connection_params, restore)
+    if connection_params and (value := connection_params.get(option)):
+        return value
+    if restore and f'restore_{option}' in data_source:
+        return data_source[f'restore_{option}']
+
+    return data_source.get(option)
+
+
+def get_hostname_from_config(data_source, connection_params=None, restore=False):
+    '''
+    Specialisation of `resolve_database_option` to handle the extra complexity of
+    the hostname option to also handle containers.
+
+    Returns a hostname/IP or raises an ValueError if a container IP lookup fails.
+    '''
+    # connection params win, full stop
+    if connection_params:
+        if container := connection_params.get('container'):
+            return get_ip_from_container(container)
+        if hostname := connection_params.get('hostname'):
+            return hostname
+    # ... then try the restore config
+    if restore:
+        if 'restore_container' in data_source:
+            return get_ip_from_container(data_source['restore_container'])
+        if 'restore_hostname' in data_source:
+            return data_source['restore_hostname']
+    # ... and finally fall back to the normal options
+    if 'container' in data_source:
+        return get_ip_from_container(data_source['container'])
+
+    return data_source.get('hostname')
+
+
+def get_ip_from_container(container):
+    '''
+    Determine the IP for a given container name via podman and docker.
+
+    Returns an IP or raises a ValueError if the lookup fails.
+    '''
+    engines = (shutil.which(engine) for engine in ('docker', 'podman'))
+    engines = [engine for engine in engines if engine]
+
+    if not engines:
+        raise ValueError("Neither 'docker' nor 'podman' could be found on the system")
+
+    last_error = None
+    for engine in engines:
+        try:
+            output = execute_command_and_capture_output(
+                (
+                    engine,
+                    'container',
+                    'inspect',
+                    '--format={{json .NetworkSettings}}',
+                    container,
+                )
+            )
+        except subprocess.CalledProcessError as error:
+            last_error = error
+            logger.debug(f"Could not find container '{container}' with engine '{engine}'")
+            continue  # Container does not exist
+
+        try:
+            network_data = json.loads(output.strip())
+        except json.JSONDecodeError as e:
+            raise ValueError(f'Could not decode JSON output from {engine}') from e
+        if main_ip := network_data.get('IPAddress'):
+            return main_ip
+        # No main IP found, try the networks
+        for network in network_data.get('Networks', {}).values():
+            if ip := network.get('IPAddress'):
+                return ip
+
+    if last_error:
+        raise last_error
+
+    raise ValueError(
+        f"Could not determine ip address for container '{container}'; running in host mode or userspace networking?"
+    )

borgmatic/hooks/data_source/dump.py

@@ -19,7 +19,9 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
     return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
 
 
-def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
+def make_data_source_dump_filename(
+    dump_path, name, hostname=None, port=None, container=None, label=None
+):
     '''
     Based on the given dump directory path, data source name, hostname, and port, return a filename
     to use for the data source dump. The hostname defaults to localhost.
@@ -29,11 +31,14 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
     if os.path.sep in name:
         raise ValueError(f'Invalid data source name {name}')
 
-    return os.path.join(
-        dump_path,
-        (hostname or 'localhost') + ('' if port is None else f':{port}'),
-        name,
-    )
+    if container:
+        hostname = container
+    elif not hostname:
+        hostname = 'localhost'
+
+    identifier = label if label else hostname + ('' if port is None else f':{port}')
+
+    return os.path.join(dump_path, identifier, name)
 
 
 def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):

borgmatic/hooks/data_source/mariadb.py

@@ -12,6 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -122,6 +123,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     show_command = (
         mariadb_show_command
         + (
@@ -130,9 +132,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -176,8 +178,10 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
@@ -192,6 +196,7 @@ def execute_dump_command(
     )
     extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     dump_command = (
         mariadb_dump_command
         + (
@@ -201,9 +206,9 @@ def execute_dump_command(
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -302,6 +307,8 @@ def dump_data_sources(
                         database_name,
                         database.get('hostname', 'localhost'),
                         database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
                     )
                 )
                 renamed_database = copy.copy(database)
@@ -326,6 +333,8 @@ def dump_data_sources(
                     database['name'],
                     database.get('hostname', 'localhost'),
                     database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
                 )
             )
             processes.append(
@@ -384,16 +393,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            hostname='*',
+            label='*',
         ),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_source_directory),
             name,
-            hostname='*',
+            label='*',
         ),
     )
 
@@ -414,25 +423,22 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname',
-        data_source.get('hostname'),
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or data_source.get('restore_password', data_source.get('password'))
+        database_config.resolve_database_option(
+            'password', data_source, connection_params, restore=True
         ),
         config,
     )

borgmatic/hooks/data_source/mongodb.py

@@ -6,6 +6,7 @@ import borgmatic.borg.pattern
 import borgmatic.config.paths
 import borgmatic.hooks.credential.parse
 from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -63,14 +64,18 @@ def dump_data_sources(
                 name,
                 database.get('hostname', 'localhost'),
                 database.get('port'),
+                database.get('label'),
+                database.get('container'),
             )
         )
 
         dump_filename = dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            database.get('hostname'),
-            database.get('port'),
+            hostname=database.get('hostname'),
+            port=database.get('port'),
+            container=database.get('container'),
+            label=database.get('label'),
         )
         dump_format = database.get('format', 'archive')
 
@@ -138,10 +143,11 @@ def build_dump_command(database, config, dump_filename, dump_format):
     dump_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     return (
         dump_command
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
-        + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+        + (('--host', shlex.quote(hostname)) if hostname else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
         + (
             (
@@ -158,7 +164,10 @@ def build_dump_command(database, config, dump_filename, dump_format):
         )
         + (('--config', make_password_config_file(password)) if password else ())
         + (
-            ('--authenticationDatabase', shlex.quote(database['authentication_database']))
+            (
+                '--authenticationDatabase',
+                shlex.quote(database['authentication_database']),
+            )
             if 'authentication_database' in database
             else ()
         )
@@ -200,16 +209,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            hostname='*',
+            label='*',
         ),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_source_directory),
             name,
-            hostname='*',
+            label='*',
         ),
     )
 
@@ -237,7 +246,10 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=data_source.get('hostname'),
+        port=data_source.get('port'),
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     restore_command = build_restore_command(
         extract_process,
@@ -247,7 +259,7 @@ def restore_data_source_dump(
         connection_params,
     )
 
-    logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
+    logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}')
     if dry_run:
         return
 
@@ -265,22 +277,21 @@ def build_restore_command(extract_process, database, config, dump_filename, conn
     '''
     Return the custom mongorestore_command from a single database configuration.
     '''
-    hostname = connection_params['hostname'] or database.get(
-        'restore_hostname',
-        database.get('hostname'),
+    hostname = database_config.resolve_database_option(
+        'hostname', database, connection_params, restore=True
+    )
+    port = database_config.resolve_database_option(
+        'port', database, connection_params, restore=True
     )
-    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or database.get('restore_username', database.get('username'))
+        database_config.resolve_database_option(
+            'username', database, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or database.get('restore_password', database.get('password'))
+        database_config.resolve_database_option(
+            'password', database, connection_params, restore=True
         ),
         config,
     )

borgmatic/hooks/data_source/mysql.py

@@ -12,6 +12,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -47,6 +48,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     show_command = (
         mysql_show_command
         + (
@@ -59,9 +61,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
             else ()
         )
         + extra_options
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -102,8 +104,10 @@ def execute_dump_command(
     dump_filename = dump.make_data_source_dump_filename(
         dump_path,
         database['name'],
-        database.get('hostname'),
-        database.get('port'),
+        hostname=database.get('hostname'),
+        port=database.get('port'),
+        container=database.get('container'),
+        label=database.get('label'),
     )
 
     if os.path.exists(dump_filename):
@@ -119,6 +123,7 @@ def execute_dump_command(
         borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
     )
     password_transport = database.get('password_transport', 'pipe')
+    hostname = database_config.resolve_database_option('hostname', database)
     dump_command = (
         mysql_dump_command
         + (
@@ -132,9 +137,9 @@ def execute_dump_command(
         )
         + extra_options
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--protocol', 'tcp') if hostname or 'port' in database else ())
         + (('--user', username) if username and password_transport == 'environment' else ())
         + (('--ssl',) if database.get('tls') is True else ())
         + (('--skip-ssl',) if database.get('tls') is False else ())
@@ -233,6 +238,8 @@ def dump_data_sources(
                         database_name,
                         database.get('hostname', 'localhost'),
                         database.get('port'),
+                        database.get('label'),
+                        database.get('container'),
                     )
                 )
                 renamed_database = copy.copy(database)
@@ -257,6 +264,8 @@ def dump_data_sources(
                     database['name'],
                     database.get('hostname', 'localhost'),
                     database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
                 )
             )
             processes.append(
@@ -315,16 +324,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            hostname='*',
+            label='*',
         ),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_source_directory),
             name,
-            hostname='*',
+            label='*',
         ),
     )
 
@@ -345,25 +354,22 @@ def restore_data_source_dump(
     subprocess.Popen) to produce output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname',
-        data_source.get('hostname'),
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
-    tls = data_source.get('restore_tls', data_source.get('tls'))
+    tls = database_config.resolve_database_option('tls', data_source, restore=True)
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
     password = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['password']
-            or data_source.get('restore_password', data_source.get('password'))
+        database_config.resolve_database_option(
+            'password', data_source, connection_params, restore=True
         ),
         config,
     )

borgmatic/hooks/data_source/postgresql.py

@@ -13,6 +13,7 @@ from borgmatic.execute import (
     execute_command_and_capture_output,
     execute_command_with_processes,
 )
+from borgmatic.hooks.data_source import config as database_config
 from borgmatic.hooks.data_source import dump
 
 logger = logging.getLogger(__name__)
@@ -32,22 +33,15 @@ def make_environment(database, config, restore_connection_params=None):
     '''
     environment = dict(os.environ)
 
-    try:
-        if restore_connection_params:
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                (
-                    restore_connection_params.get('password')
-                    or database.get('restore_password', database['password'])
-                ),
-                config,
-            )
-        else:
-            environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
-                database['password'],
-                config,
-            )
-    except (AttributeError, KeyError):
-        pass
+    password = database_config.resolve_database_option(
+        'password', database, restore_connection_params, restore=restore_connection_params
+    )
+
+    if password:
+        environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
+            password,
+            config,
+        )
 
     if 'ssl_mode' in database:
         environment['PGSSLMODE'] = database['ssl_mode']
@@ -91,10 +85,11 @@ def database_names_to_dump(database, config, environment, dry_run):
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
     )
+    hostname = database_config.resolve_database_option('hostname', database)
     list_command = (
         psql_command
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--host', hostname) if hostname else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (
             (
@@ -172,6 +167,8 @@ def dump_data_sources(
                     database_name,
                     database.get('hostname', 'localhost'),
                     database.get('port'),
+                    database.get('label'),
+                    database.get('container'),
                 )
             )
             dump_format = database.get('format', None if database_name == 'all' else 'custom')
@@ -184,8 +181,10 @@ def dump_data_sources(
             dump_filename = dump.make_data_source_dump_filename(
                 dump_path,
                 database_name,
-                database.get('hostname'),
-                database.get('port'),
+                hostname=database.get('hostname'),
+                port=database.get('port'),
+                container=database.get('container'),
+                label=database.get('label'),
             )
 
             if os.path.exists(dump_filename):
@@ -194,6 +193,7 @@ def dump_data_sources(
                 )
                 continue
 
+            hostname = database_config.resolve_database_option('hostname', database)
             command = (
                 dump_command
                 + (
@@ -201,7 +201,7 @@ def dump_data_sources(
                     '--clean',
                     '--if-exists',
                 )
-                + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+                + (('--host', shlex.quote(hostname)) if hostname else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
                     (
@@ -302,16 +302,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            hostname='*',
+            label='*',
         ),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_source_directory),
             name,
-            hostname='*',
+            label='*',
         ),
     )
 
@@ -339,17 +339,15 @@ def restore_data_source_dump(
     hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
-    hostname = connection_params['hostname'] or data_source.get(
-        'restore_hostname',
-        data_source.get('hostname'),
+    hostname = database_config.resolve_database_option(
+        'hostname', data_source, connection_params, restore=True
     )
-    port = str(
-        connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
+    port = database_config.resolve_database_option(
+        'port', data_source, connection_params, restore=True
     )
     username = borgmatic.hooks.credential.parse.resolve_credential(
-        (
-            connection_params['username']
-            or data_source.get('restore_username', data_source.get('username'))
+        database_config.resolve_database_option(
+            'username', data_source, connection_params, restore=True
         ),
         config,
     )
@@ -358,7 +356,10 @@ def restore_data_source_dump(
     dump_filename = dump.make_data_source_dump_filename(
         make_dump_path(borgmatic_runtime_directory),
         data_source['name'],
-        data_source.get('hostname'),
+        hostname=hostname,
+        port=port,
+        container=data_source.get('container'),
+        label=data_source.get('label'),
     )
     psql_command = tuple(
         shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
@@ -367,7 +368,7 @@ def restore_data_source_dump(
         psql_command
         + ('--no-password', '--no-psqlrc', '--quiet')
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (
@@ -388,7 +389,7 @@ def restore_data_source_dump(
         + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
         + (('--dbname', data_source['name']) if not all_databases else ())
         + (('--host', hostname) if hostname else ())
-        + (('--port', port) if port else ())
+        + (('--port', str(port)) if port else ())
         + (('--username', username) if username else ())
         + (('--no-owner',) if data_source.get('no_owner', False) else ())
         + (

borgmatic/hooks/data_source/sqlite.py

@@ -57,8 +57,7 @@ def dump_data_sources(
         database_path = database['path']
         dumps_metadata.append(
             borgmatic.actions.restore.Dump(
-                'sqlite_databases',
-                database['name'],
+                'sqlite_databases', database['name'], label=database.get('label')
             )
         )
 
@@ -71,7 +70,9 @@ def dump_data_sources(
             )
 
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        dump_filename = dump.make_data_source_dump_filename(dump_path, database['name'])
+        dump_filename = dump.make_data_source_dump_filename(
+            dump_path, database['name'], label=database.get('label')
+        )
 
         if os.path.exists(dump_filename):
             logger.warning(
@@ -143,16 +144,16 @@ def make_data_source_dump_patterns(
     borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
 
     return (
-        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
+        dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_runtime_directory),
             name,
-            hostname='*',
+            label='*',
         ),
         dump.make_data_source_dump_filename(
             make_dump_path(borgmatic_source_directory),
             name,
-            hostname='*',
+            label='*',
         ),
     )
 

docs/how-to/backup-your-databases.md

@@ -72,8 +72,10 @@ Here's a more involved example that connects to remote databases:
 ```yaml
 postgresql_databases:
     - name: users
+      label: database_server1
       hostname: database1.example.org
     - name: orders
+      label: database_server2
       hostname: database2.example.org
       port: 5433
       username: postgres
@@ -212,34 +214,29 @@ these options in the `hooks:` section of your configuration.
 
 #### Database client on the host
 
-But what if borgmatic is running on the host? You can still connect to a
-database server container if its ports are properly exposed to the host. For
-instance, when running the database container, you can specify `--publish
-127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433
-on the host (only reachable on localhost, in this case). Or the same thing with
-Docker Compose:
+But what if borgmatic is running on the host?
 
-```yaml
-services:
-   your-database-server-container-name:
-       image: postgres
-       ports:
-           - 127.0.0.1:5433:5432
-```
-
-And then you can configure borgmatic running on the host to connect to the
-database:
+<span class="minilink minilink-addedin">New in version 2.0.8</span> You can
+connect to the database container by specifying its container name or ID:
 
 ```yaml
 postgresql_databases:
     - name: users
-      hostname: 127.0.0.1
+      container: your-database-server-container-name
       port: 5433
       username: postgres
       password: trustsome1
 ```
 
-Alter the ports in these examples to suit your particular database system.
+borgmatic uses the `docker`/`podman` CLI to figure out the container IP to
+connect to. But `container:` does not work when borgmatic itself is running in a
+container; in that case, use `hostname:` as described above.
+
+<span class="minilink minilink-addedin">Prior to version 2.0.8</span> If you're
+running an older version of borgmatic on the host, you can publish your database
+container ports to the host (e.g. via `docker run --publish` or Compose's
+`ports`)—and then configure borgmatic to connect to `localhost` and the
+published port.
 
 
 #### Database client in a running container

tests/end-to-end/commands/docker

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "{\"IPAddress\": \"$4\"}"

tests/end-to-end/hooks/data_source/test_database.py

@@ -94,6 +94,63 @@ sqlite_databases:
     return ruamel.yaml.YAML(typ='safe').load(config_yaml)
 
 
+def write_container_configuration(
+    source_directory,
+    config_path,
+    repository_path,
+    user_runtime_directory,
+):
+    '''
+    Write out borgmatic configuration into a file at the config path. Set the options so as to work
+    for testing. This includes injecting the given repository path, borgmatic source directory for
+    storing database dumps, and encryption passphrase.
+    '''
+
+    config_yaml = f'''
+source_directories:
+    - {source_directory}
+repositories:
+    - path: {repository_path}
+user_runtime_directory: {user_runtime_directory}
+
+encryption_passphrase: "test"
+
+postgresql_databases:
+    - name: test
+      container: postgresql
+      username: postgres
+      password: test
+      restore_container: postgresql2
+      restore_port: 5433
+      restore_password: test2
+mariadb_databases:
+    - name: test
+      container: mariadb
+      username: root
+      password: test
+      restore_container: mariadb2
+      restore_port: 3307
+      restore_username: root
+      restore_password: test2
+mysql_databases:
+    - name: test
+      container: not-actually-mysql
+      username: root
+      password: test
+mongodb_databases:
+    - name: test
+      container: mongodb
+      username: root
+      password: test
+      authentication_database: admin
+'''
+
+    with open(config_path, 'w') as config_file:
+        config_file.write(config_yaml)
+
+    return ruamel.yaml.YAML(typ='safe').load(config_yaml)
+
+
 @pytest.mark.parametrize(
     'postgresql_all_dump_format,mariadb_mysql_all_dump_format',
     (
@@ -116,6 +173,7 @@ def write_custom_restore_configuration(
     for testing with custom restore options. This includes a custom restore_hostname, restore_port,
     restore_username, restore_password and restore_path.
     '''
+
     config_yaml = f'''
 source_directories:
     - {source_directory}
@@ -212,8 +270,13 @@ postgresql_databases:
 
 
 def get_connection_params(database, use_restore_options=False):
-    hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get(
-        'hostname',
+    hostname = (
+        database.get('restore_container', database.get('restore_hostname'))
+        if use_restore_options
+        else None
+    ) or database.get(
+        'container',
+        database.get('hostname'),
     )
     port = (database.get('restore_port') if use_restore_options else None) or database.get('port')
     username = (database.get('restore_username') if use_restore_options else None) or database.get(
@@ -656,3 +719,65 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_containers():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+
+    original_working_directory = os.getcwd()
+    original_path = os.environ.get('PATH', '')
+
+    os.environ['PATH'] = f'/app/tests/end-to-end/commands:{original_path}'
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        config = write_container_configuration(
+            temporary_directory,
+            config_path,
+            repository_path,
+            temporary_directory,
+        )
+        create_test_tables(config)
+        select_test_tables(config)
+
+        subprocess.check_call(
+            [
+                'borgmatic',
+                '-v',
+                '2',
+                '--config',
+                config_path,
+                'repo-create',
+                '--encryption',
+                'repokey',
+            ],
+        )
+
+        # Run borgmatic to generate a backup archive including database dumps.
+        subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
+
+        # Get the created archive name.
+        output = subprocess.check_output(
+            ['borgmatic', '--config', config_path, 'list', '--json'],
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Restore the databases from the archive.
+        drop_test_tables(config)
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name],
+        )
+
+        # Ensure the test tables have actually been restored.
+        select_test_tables(config, use_restore_options=True)
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+        drop_test_tables(config)
+        os.environ['PATH'] = original_path

tests/integration/hooks/data_source/test_config.py

@@ -0,0 +1,54 @@
+from flexmock import flexmock
+
+from borgmatic.hooks.data_source import config as module
+
+
+def test_resolve_database_option_gets_hostname_from_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'original_container'
+    ).and_return('container_ip_1')
+
+    assert module.resolve_database_option('hostname', data_source) == 'container_ip_1'
+
+
+def test_resolve_database_option_gets_hostname_from_connection_params_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+    connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'}
+
+    flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'connection_container'
+    ).and_return('container_ip_2')
+
+    assert (
+        module.resolve_database_option('hostname', data_source, connection_params)
+        == 'container_ip_2'
+    )
+
+
+def test_resolve_database_option_gets_hostname_from_restore_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+
+    flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'restore_container'
+    ).and_return('container_ip_3')
+
+    assert module.resolve_database_option('hostname', data_source, restore=True) == 'container_ip_3'

tests/unit/actions/test_restore.py

@@ -126,6 +126,24 @@ import borgmatic.actions.restore as module
             5432,
             False,
         ),
+        (
+            module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'),
+            module.Dump('postgresql_databases', 'foo', 'some_host2', None, 'unique'),
+            5432,
+            True,
+        ),
+        (
+            module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'),
+            module.Dump(module.UNSPECIFIED, 'foo', 'some_host2', None, 'unique'),
+            5432,
+            True,
+        ),
+        (
+            module.Dump('postgresql_databases', 'foo', container='container'),
+            module.Dump('postgresql_databases', 'foo', container='container'),
+            5432,
+            True,
+        ),
     ),
 )
 def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
@@ -172,6 +190,14 @@ def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
             module.Dump('postgresql_databases', 'foo', 'host', module.UNSPECIFIED),
             'foo@host (postgresql_databases)',
         ),
+        (
+            module.Dump('postgresql_databases', 'foo', 'host', 1234, 'label'),
+            'foo@label (postgresql_databases)',
+        ),
+        (
+            module.Dump('postgresql_databases', 'foo', container='container'),
+            'foo@container (postgresql_databases)',
+        ),
         (
             module.Dump(
                 module.UNSPECIFIED,
@@ -192,7 +218,7 @@ def test_get_configured_data_source_matches_data_source_with_restore_dump():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port)
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'bar'),
+        module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED),
         module.Dump('postgresql_databases', 'bar'),
         default_port=default_port,
     ).and_return(True)
@@ -239,7 +265,7 @@ def test_get_configured_data_source_with_multiple_matching_data_sources_errors()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port)
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'bar'),
+        module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED),
         module.Dump('postgresql_databases', 'bar'),
         default_port=default_port,
     ).and_return(True)
@@ -720,11 +746,23 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'bar',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'bar'),
     ).and_return(True)
 
@@ -734,6 +772,8 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
             data_sources=['foo', 'bar'],
             original_hostname=None,
             original_port=None,
+            original_label=None,
+            original_container=None,
         ),
         dumps_from_archive=dumps_from_archive,
     ) == {
@@ -747,18 +787,31 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive():
         module.Dump('postgresql_databases', 'foo'),
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
-    flexmock(module).should_receive('render_dump_metadata').and_return('test')
+    flexmock(module).should_receive('dumps_match').with_args(
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
+        module.Dump('postgresql_databases', 'foo'),
+    ).and_return(True)
+    flexmock(module).should_receive('render_dump_metadata').and_return('test').once()
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         module.get_dumps_to_restore(
             restore_arguments=flexmock(
                 hook=None,
                 data_sources=['foo', 'bar'],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
             dumps_from_archive=dumps_from_archive,
         )
+    assert 'dump test missing from archive' in str(exc_info.value)
 
 
 def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps():
@@ -775,6 +828,8 @@ def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps():
                 data_sources=[],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
             dumps_from_archive=dumps_from_archive,
         )
@@ -789,11 +844,23 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'bar',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'bar'),
     ).and_return(True)
 
@@ -804,6 +871,8 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
                 data_sources=['all'],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
             dumps_from_archive=dumps_from_archive,
         )
@@ -818,11 +887,23 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'bar',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'bar'),
     ).and_return(True)
 
@@ -833,6 +914,8 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
                 data_sources=['all', 'foo', 'bar'],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
             dumps_from_archive=dumps_from_archive,
         )
@@ -843,48 +926,72 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
 def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive():
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('mariadb_databases', 'foo'),
     ).and_return(True)
     flexmock(module).should_receive('render_dump_metadata').and_return('test')
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         module.get_dumps_to_restore(
             restore_arguments=flexmock(
                 hook=None,
                 data_sources=['foo'],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
             dumps_from_archive={
                 module.Dump('postgresql_databases', 'foo'),
                 module.Dump('mariadb_databases', 'foo'),
             },
         )
+    assert 'Try adding flags to disambiguate.' in str(exc_info.value)
 
 
 def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_dumps_missing_from_archive():
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            module.UNSPECIFIED,
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
-    flexmock(module).should_receive('render_dump_metadata').and_return('test')
+    flexmock(module).should_receive('render_dump_metadata').and_return('test').once()
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         module.get_dumps_to_restore(
             restore_arguments=flexmock(
                 hook=None,
                 data_sources=['all', 'foo', 'bar'],
                 original_hostname=None,
                 original_port=None,
+                original_label=None,
+                original_container=None,
             ),
-            dumps_from_archive={module.Dump('postresql_databases', 'foo')},
+            dumps_from_archive={module.Dump('postgresql_databases', 'foo')},
         )
+    assert 'dump test missing from archive' in str(exc_info.value)
 
 
 def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_archive():
@@ -895,7 +1002,13 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            'postgresql_databases',
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
 
@@ -905,6 +1018,8 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
             data_sources=['foo'],
             original_hostname=None,
             original_port=None,
+            original_label=None,
+            original_container=None,
         ),
         dumps_from_archive=dumps_from_archive,
     ) == {
@@ -920,7 +1035,13 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED),
+        module.Dump(
+            'postgresql_databases',
+            'foo',
+            hostname=module.UNSPECIFIED,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo'),
     ).and_return(True)
 
@@ -930,6 +1051,8 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
             data_sources=['foo'],
             original_hostname=None,
             original_port=None,
+            original_label=None,
+            original_container=None,
         ),
         dumps_from_archive=dumps_from_archive,
     ) == {
@@ -945,7 +1068,13 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'foo', 'host'),
+        module.Dump(
+            'postgresql_databases',
+            'foo',
+            'host',
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo', 'host'),
     ).and_return(True)
 
@@ -955,6 +1084,8 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
             data_sources=['foo'],
             original_hostname='host',
             original_port=None,
+            original_label=None,
+            original_container=None,
         ),
         dumps_from_archive=dumps_from_archive,
     ) == {
@@ -970,7 +1101,14 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
     }
     flexmock(module).should_receive('dumps_match').and_return(False)
     flexmock(module).should_receive('dumps_match').with_args(
-        module.Dump('postgresql_databases', 'foo', 'host', 1234),
+        module.Dump(
+            'postgresql_databases',
+            'foo',
+            'host',
+            1234,
+            label=module.UNSPECIFIED,
+            container=module.UNSPECIFIED,
+        ),
         module.Dump('postgresql_databases', 'foo', 'host', 1234),
     ).and_return(True)
 
@@ -980,6 +1118,8 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
             data_sources=['foo'],
             original_hostname='host',
             original_port=1234,
+            original_label=None,
+            original_container=None,
         ),
         dumps_from_archive=dumps_from_archive,
     ) == {
@@ -1087,6 +1227,7 @@ def test_run_restore_restores_each_data_source():
             username=None,
             password=None,
             restore_path=None,
+            container=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -1171,6 +1312,7 @@ def test_run_restore_restores_data_source_by_falling_back_to_all_name():
             username=None,
             password=None,
             restore_path=None,
+            container=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -1252,6 +1394,7 @@ def test_run_restore_restores_data_source_configured_with_all_name():
             username=None,
             password=None,
             restore_path=None,
+            container=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -1333,6 +1476,7 @@ def test_run_restore_skips_missing_data_source():
             username=None,
             password=None,
             restore_path=None,
+            container=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),
@@ -1410,6 +1554,7 @@ def test_run_restore_restores_data_sources_from_different_hooks():
             username=None,
             password=None,
             restore_path=None,
+            container=None,
         ),
         global_arguments=flexmock(dry_run=False),
         local_path=flexmock(),

tests/unit/config/test_arguments.py

@@ -209,6 +209,19 @@ def test_prepare_arguments_for_config_skips_option_with_none_value():
     ) == ((('other_option',), 'value2'),)
 
 
+def test_prepare_arguments_for_config_skips_option_with_complex_schema():
+    assert module.prepare_arguments_for_config(
+        global_arguments=flexmock(my_option='value1', other_option='value2'),
+        schema={
+            'type': 'object',
+            'properties': {
+                'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}},
+                'other_option': {'type': 'string'},
+            },
+        },
+    ) == ((('other_option',), 'value2'),)
+
+
 def test_prepare_arguments_for_config_skips_option_missing_from_schema():
     assert module.prepare_arguments_for_config(
         global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}),

tests/unit/hooks/data_source/test_config.py

@@ -0,0 +1,157 @@
+from subprocess import CalledProcessError
+
+import pytest
+from flexmock import flexmock
+
+from borgmatic.hooks.data_source import config as module
+
+
+def test_resolve_database_option_uses_config_value():
+    data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
+
+    assert module.resolve_database_option('option', data_source) == 'original_value'
+
+
+def test_resolve_database_option_with_restore_uses_restore_value():
+    data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
+
+    assert module.resolve_database_option('option', data_source, restore=True) == 'restore_value'
+
+
+def test_resolve_database_option_with_connection_params_uses_connection_params_value():
+    data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
+    connection_params = {'option': 'connection_value'}
+
+    assert (
+        module.resolve_database_option('option', data_source, connection_params)
+        == 'connection_value'
+    )
+
+
+def test_resolve_database_option_with_restore_and_connection_params_uses_connection_params_value():
+    data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
+    connection_params = {'option': 'connection_value'}
+
+    assert (
+        module.resolve_database_option('option', data_source, connection_params, restore=True)
+        == 'connection_value'
+    )
+
+
+def test_resolve_database_option_with_hostname_uses_hostname_specific_function():
+    data_source = {'hostname': 'original_value'}
+    connection_params = {'hostname': 'connection_value'}
+
+    flexmock(module).should_receive('get_hostname_from_config').and_return('special_value').once()
+
+    assert (
+        module.resolve_database_option('hostname', data_source, connection_params, restore=True)
+        == 'special_value'
+    )
+
+
+def test_get_hostname_from_config_gets_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'original_container'
+    ).and_return('container_ip_1')
+
+    assert module.get_hostname_from_config(data_source) == 'container_ip_1'
+
+
+def test_get_hostname_from_config_gets_connection_params_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+    connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'}
+
+    flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'connection_container'
+    ).and_return('container_ip_2')
+
+    assert module.get_hostname_from_config(data_source, connection_params) == 'container_ip_2'
+
+
+def test_get_hostname_from_config_gets_restore_container_ip():
+    data_source = {
+        'container': 'original_container',
+        'hostname': 'original_hostname',
+        'restore_container': 'restore_container',
+        'restore_hostname': 'restore_hostname',
+    }
+
+    flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
+    flexmock(module).should_receive('get_ip_from_container').with_args(
+        'restore_container'
+    ).and_return('container_ip_3')
+
+    assert module.get_hostname_from_config(data_source, restore=True) == 'container_ip_3'
+
+
+def test_get_ip_from_container_without_engines_errors():
+    flexmock(module.shutil).should_receive('which').and_return(None).and_return(None)
+
+    with pytest.raises(ValueError):
+        module.get_ip_from_container('yolo')
+
+
+def test_get_ip_from_container_parses_top_level_ip_address():
+    flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
+
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '{"IPAddress": "1.2.3.4"}'
+    )
+
+    assert module.get_ip_from_container('yolo') == '1.2.3.4'
+
+
+def test_get_ip_from_container_parses_network_ip_address():
+    flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
+
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}'
+    )
+
+    assert module.get_ip_from_container('yolo') == '5.6.7.8'
+
+
+def test_get_ip_from_container_without_container_errors():
+    flexmock(module.shutil).should_receive('which').and_return('/usr/bin/podman')
+    flexmock(module).should_receive('execute_command_and_capture_output').and_raise(
+        CalledProcessError, 1, ['/usr/bin/podman', 'inspect', 'yolo'], None, 'No such object'
+    )
+
+    with pytest.raises(CalledProcessError):
+        module.get_ip_from_container('does not exist')
+
+
+def test_get_ip_from_container_without_network_errors():
+    flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
+
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return('{}')
+
+    with pytest.raises(ValueError) as exc_info:
+        module.get_ip_from_container('yolo')
+
+    assert 'Could not determine ip address for container' in str(exc_info.value)
+
+
+def test_get_ip_from_container_with_broken_output_errors():
+    flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
+
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return('abc')
+
+    with pytest.raises(ValueError) as exc_info:
+        module.get_ip_from_container('yolo')
+
+    assert 'Could not decode JSON output' in str(exc_info.value)

tests/unit/hooks/data_source/test_dump.py

@@ -25,6 +25,24 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port():
     )
 
 
+def test_make_data_source_dump_filename_uses_label():
+    assert (
+        module.make_data_source_dump_filename(
+            'databases', 'test', 'hostname', 1234, label='custom_label'
+        )
+        == 'databases/custom_label/test'
+    )
+
+
+def test_make_data_source_dump_filename_uses_container():
+    assert (
+        module.make_data_source_dump_filename(
+            'databases', 'test', 'hostname', 1234, container='container'
+        )
+        == 'databases/container:1234/test'
+    )
+
+
 def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost():
     assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test'
 
@@ -51,7 +69,7 @@ def test_write_data_source_dumps_metadata_writes_json_to_file():
 
     assert (
         dumps_stream.getvalue()
-        == '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "port": null}]}'
+        == '{"dumps": [{"container": null, "data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}, {"container": null, "data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}]}'
     )