Load credentials from database hooks (#966).

borgmatic/borg/environment.py

@@ -41,7 +41,7 @@ def make_environment(config):
         value = config.get(option_name)
 
         if option_name in CREDENTIAL_OPTIONS:
-            value = borgmatic.hooks.credential.tag.resolve_credential(value) 
+            value = borgmatic.hooks.credential.tag.resolve_credential(value)
 
         if value is not None:
             environment[environment_variable_name] = str(value)

borgmatic/commands/borgmatic.py

@@ -533,9 +533,7 @@ def run_actions(
     )
 
 
-def load_configurations(
-    config_filenames, overrides=None, resolve_env=True
-):
+def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''
     Given a sequence of configuration filenames, a sequence of configuration file override strings
     in the form of "option.suboption=value", and whether to resolve environment variables, load and

borgmatic/config/validate.py

@@ -84,9 +84,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(
-    config_filename, schema_filename, overrides=None, resolve_env=True
-):
+def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form

borgmatic/hooks/credential/tag.py

@@ -22,8 +22,6 @@ def resolve_credential(tag):
         except ValueError:
             raise ValueError(f'Cannot load credential with invalid syntax "{tag}"')
 
-        return borgmatic.hooks.dispatch.call_hook(
-            'load_credential', {}, hook_name, credential_name
-        )
+        return borgmatic.hooks.dispatch.call_hook('load_credential', {}, hook_name, credential_name)
 
     return tag

borgmatic/hooks/data_source/mariadb.py

@@ -5,6 +5,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.tag
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -45,7 +46,11 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username']))
+            if 'username' in database
+            else ()
+        )
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
@@ -96,7 +101,11 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username']))
+            if 'username' in database
+            else ()
+        )
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -152,7 +161,11 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        extra_environment = (
+            {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])}
+            if 'password' in database
+            else None
+        )
         dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
 
         if not dump_database_names:
@@ -251,11 +264,13 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['username']
+        or data_source.get('restore_username', data_source.get('username'))
     )
-    password = connection_params['password'] or data_source.get(
-        'restore_password', data_source.get('password')
+    password = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['password']
+        or data_source.get('restore_password', data_source.get('password'))
     )
 
     mariadb_restore_command = tuple(
@@ -274,7 +289,11 @@ def restore_data_source_dump(
         + (('--protocol', 'tcp') if hostname or port else ())
         + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    extra_environment = (
+        {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)}
+        if password
+        else None
+    )
 
     logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/data_source/mongodb.py

@@ -4,6 +4,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.tag
 from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks.data_source import dump
 
@@ -98,8 +99,26 @@ def build_dump_command(database, dump_filename, dump_format):
         + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
         + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
         + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
-        + (('--username', shlex.quote(database['username'])) if 'username' in database else ())
-        + (('--password', shlex.quote(database['password'])) if 'password' in database else ())
+        + (
+            (
+                '--username',
+                shlex.quote(
+                    borgmatic.hooks.credential.tag.resolve_credential(database['username'])
+                ),
+            )
+            if 'username' in database
+            else ()
+        )
+        + (
+            (
+                '--password',
+                shlex.quote(
+                    borgmatic.hooks.credential.tag.resolve_credential(database['password'])
+                ),
+            )
+            if 'password' in database
+            else ()
+        )
         + (
             ('--authenticationDatabase', shlex.quote(database['authentication_database']))
             if 'authentication_database' in database
@@ -198,11 +217,11 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
         'restore_hostname', database.get('hostname')
     )
     port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
-    username = connection_params['username'] or database.get(
-        'restore_username', database.get('username')
+    username = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['username'] or database.get('restore_username', database.get('username'))
     )
-    password = connection_params['password'] or database.get(
-        'restore_password', database.get('password')
+    password = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['password'] or database.get('restore_password', database.get('password'))
     )
 
     command = ['mongorestore']
@@ -217,9 +236,9 @@ def build_restore_command(extract_process, database, dump_filename, connection_p
     if port:
         command.extend(('--port', str(port)))
     if username:
-        command.extend(('--username', username))
+        command.extend(('--username', borgmatic.hooks.credential.tag.resolve_credential(username)))
     if password:
-        command.extend(('--password', password))
+        command.extend(('--password', borgmatic.hooks.credential.tag.resolve_credential(password)))
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:

borgmatic/hooks/data_source/mysql.py

@@ -5,6 +5,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.tag
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -45,7 +46,11 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username']))
+            if 'username' in database
+            else ()
+        )
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
@@ -95,7 +100,11 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (
+            ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username']))
+            if 'username' in database
+            else ()
+        )
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -151,7 +160,11 @@ def dump_data_sources(
 
     for database in databases:
         dump_path = make_dump_path(borgmatic_runtime_directory)
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        extra_environment = (
+            {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])}
+            if 'password' in database
+            else None
+        )
         dump_database_names = database_names_to_dump(database, extra_environment, dry_run)
 
         if not dump_database_names:
@@ -250,11 +263,13 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['username']
+        or data_source.get('restore_username', data_source.get('username'))
     )
-    password = connection_params['password'] or data_source.get(
-        'restore_password', data_source.get('password')
+    password = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['password']
+        or data_source.get('restore_password', data_source.get('password'))
     )
 
     mysql_restore_command = tuple(
@@ -273,7 +288,11 @@ def restore_data_source_dump(
         + (('--protocol', 'tcp') if hostname or port else ())
         + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': password} if password else None
+    extra_environment = (
+        {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)}
+        if password
+        else None
+    )
 
     logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/data_source/postgresql.py

@@ -7,6 +7,7 @@ import shlex
 
 import borgmatic.borg.pattern
 import borgmatic.config.paths
+import borgmatic.hooks.credential.tag
 from borgmatic.execute import (
     execute_command,
     execute_command_and_capture_output,
@@ -33,11 +34,14 @@ def make_extra_environment(database, restore_connection_params=None):
 
     try:
         if restore_connection_params:
-            extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
-                'restore_password', database['password']
+            extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential(
+                restore_connection_params.get('password')
+                or database.get('restore_password', database['password'])
             )
         else:
-            extra['PGPASSWORD'] = database['password']
+            extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential(
+                database['password']
+            )
     except (AttributeError, KeyError):
         pass
 
@@ -82,7 +86,11 @@ def database_names_to_dump(database, extra_environment, dry_run):
         + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        + (
+            ('--username', borgmatic.hooks.credential.tag.resolve_credential(database['username']))
+            if 'username' in database
+            else ()
+        )
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug('Querying for "all" PostgreSQL databases to dump')
@@ -174,7 +182,12 @@ def dump_data_sources(
                 + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
                 + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
                 + (
-                    ('--username', shlex.quote(database['username']))
+                    (
+                        '--username',
+                        shlex.quote(
+                            borgmatic.hooks.credential.tag.resolve_credential(database['username'])
+                        ),
+                    )
                     if 'username' in database
                     else ()
                 )
@@ -290,8 +303,9 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
-    username = connection_params['username'] or data_source.get(
-        'restore_username', data_source.get('username')
+    username = borgmatic.hooks.credential.tag.resolve_credential(
+        connection_params['username']
+        or data_source.get('restore_username', data_source.get('username'))
     )
 
     all_databases = bool(data_source['name'] == 'all')

borgmatic/hooks/monitoring/ntfy.py

@@ -50,9 +50,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         }
 
         try:
-            username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username'))
-            password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password'))
-            access_token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('access_token'))
+            username = borgmatic.hooks.credential.tag.resolve_credential(
+                hook_config.get('username')
+            )
+            password = borgmatic.hooks.credential.tag.resolve_credential(
+                hook_config.get('password')
+            )
+            access_token = borgmatic.hooks.credential.tag.resolve_credential(
+                hook_config.get('access_token')
+            )
         except ValueError as error:
             logger.warning(f'Ntfy credential error: {error}')
             return

borgmatic/hooks/monitoring/pagerduty.py

@@ -41,7 +41,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev
         return
 
     try:
-        inegration_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('integration_key'))
+        inegration_key = borgmatic.hooks.credential.tag.resolve_credential(
+            hook_config.get('integration_key')
+        )
     except ValueError as error:
         logger.warning(f'PagerDuty credential error: {error}')
         return