Add "tls" options to the MariaDB and MySQL database hooks.

NEWS

@@ -8,6 +8,8 @@
    "--password" on the command-line!
  * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit.
  * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification.
+ * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption
+   between client and server.
 
 1.9.12
  * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will

borgmatic/config/schema.yaml

@@ -1210,6 +1210,20 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and server. The default varies based on the
+                        MariaDB version.
+                    example: false
+                restore_tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and restore server. The default varies based on
+                        the MariaDB version.
+                    example: false
                 mariadb_dump_command:
                     type: string
                     description: |
@@ -1340,6 +1354,20 @@ properties:
                         Defaults to the "password" option. Supports the
                         "{credential ...}" syntax.
                     example: trustsome1
+                tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and server. The default varies based on the
+                        MySQL installation.
+                    example: false
+                restore_tls:
+                    type: boolean
+                    description: |
+                        Whether to TLS-encrypt data transmitted between the
+                        client and restore server. The default varies based on
+                        the MySQL installation.
+                    example: false
                 mysql_dump_command:
                     type: string
                     description: |

borgmatic/hooks/data_source/mariadb.py

@@ -125,6 +125,8 @@ def database_names_to_dump(database, config, username, password, environment, dr
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
@@ -188,6 +190,8 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -357,6 +361,7 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
+    tls = data_source.get('restore_tls', data_source.get('tls'))
     username = borgmatic.hooks.credential.parse.resolve_credential(
         (
             connection_params['username']
@@ -384,6 +389,8 @@ def restore_data_source_dump(
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--ssl',) if tls is True else ())
+        + (('--skip-ssl',) if tls is False else ())
     )
     environment = dict(os.environ)
 

borgmatic/hooks/data_source/mysql.py

@@ -54,6 +54,8 @@ def database_names_to_dump(database, config, username, password, environment, dr
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--skip-column-names', '--batch')
         + ('--execute', 'show schemas')
     )
@@ -117,6 +119,8 @@ def execute_dump_command(
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--ssl',) if database.get('tls') is True else ())
+        + (('--skip-ssl',) if database.get('tls') is False else ())
         + ('--databases',)
         + database_names
         + ('--result-file', dump_filename)
@@ -286,6 +290,7 @@ def restore_data_source_dump(
     port = str(
         connection_params['port'] or data_source.get('restore_port', data_source.get('port', ''))
     )
+    tls = data_source.get('restore_tls', data_source.get('tls'))
     username = borgmatic.hooks.credential.parse.resolve_credential(
         (
             connection_params['username']
@@ -317,6 +322,8 @@ def restore_data_source_dump(
         + (('--host', hostname) if hostname else ())
         + (('--port', str(port)) if port else ())
         + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--ssl',) if tls is True else ())
+        + (('--skip-ssl',) if tls is False else ())
     )
     environment = dict(os.environ)
 

tests/unit/hooks/data_source/test_mariadb.py

@@ -151,6 +151,64 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
     assert names == ('foo', 'bar')
 
 
+def test_database_names_to_dump_runs_mariadb_with_tls():
+    environment = flexmock()
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        'root', 'trustsome1', None
+    ).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'mariadb',
+            '--defaults-extra-file=/dev/fd/99',
+            '--ssl',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        environment=environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False
+    )
+
+    assert names == ('foo', 'bar')
+
+
+def test_database_names_to_dump_runs_mariadb_without_tls():
+    environment = flexmock()
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        'root', 'trustsome1', None
+    ).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'mariadb',
+            '--defaults-extra-file=/dev/fd/99',
+            '--skip-ssl',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        environment=environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False
+    )
+
+    assert names == ('foo', 'bar')
+
+
 def test_use_streaming_true_for_any_databases():
     assert module.use_streaming(
         databases=[flexmock(), flexmock()],
@@ -496,6 +554,94 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port():
     )
 
 
+def test_execute_dump_command_runs_mariadb_dump_with_tls():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        'root', 'trustsome1', None
+    ).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--defaults-extra-file=/dev/fd/99',
+            '--add-drop-database',
+            '--ssl',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'tls': True},
+            config={},
+            username='root',
+            password='trustsome1',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mariadb_dump_without_tls():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        'root', 'trustsome1', None
+    ).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mariadb-dump',
+            '--defaults-extra-file=/dev/fd/99',
+            '--add-drop-database',
+            '--skip-ssl',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'tls': False},
+            config={},
+            username='root',
+            password='trustsome1',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
 def test_execute_dump_command_runs_mariadb_dump_with_username_and_password():
     process = flexmock()
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -890,6 +1036,86 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
     )
 
 
+def test_restore_data_source_dump_runs_mariadb_with_tls():
+    hook_config = [{'name': 'foo', 'tls': True}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        None, None, None
+    ).and_return(())
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mariadb',
+            '--batch',
+            '--ssl',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        environment={'USER': 'root'},
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
+def test_restore_data_source_dump_runs_mariadb_without_tls():
+    hook_config = [{'name': 'foo', 'tls': False}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module).should_receive('parse_extra_options').and_return((), None)
+    flexmock(module).should_receive('make_defaults_file_options').with_args(
+        None, None, None
+    ).and_return(())
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mariadb',
+            '--batch',
+            '--skip-ssl',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        environment={'USER': 'root'},
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_restore_data_source_dump_runs_mariadb_with_username_and_password():
     hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
     extract_process = flexmock(stdout=flexmock())
@@ -990,10 +1216,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
             'password': 'trustsome1',
             'hostname': 'dbhost',
             'port': 'dbport',
+            'tls': True,
             'restore_username': 'restoreuser',
             'restore_password': 'restorepass',
             'restore_hostname': 'restorehost',
             'restore_port': 'restoreport',
+            'restore_tls': False,
         }
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -1017,6 +1245,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
             'restoreport',
             '--protocol',
             'tcp',
+            '--skip-ssl',
         ),
         processes=[extract_process],
         output_log_level=logging.DEBUG,

tests/unit/hooks/data_source/test_mysql.py

@@ -60,6 +60,68 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
     assert names == ('foo', 'bar')
 
 
+def test_database_names_to_dump_runs_mysql_with_tls():
+    environment = flexmock()
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'mysql',
+            '--defaults-extra-file=/dev/fd/99',
+            '--ssl',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        environment=environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False
+    )
+
+    assert names == ('foo', 'bar')
+
+
+def test_database_names_to_dump_runs_mysql_without_tls():
+    environment = flexmock()
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        (
+            'mysql',
+            '--defaults-extra-file=/dev/fd/99',
+            '--skip-ssl',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        environment=environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False
+    )
+
+    assert names == ('foo', 'bar')
+
+
 def test_use_streaming_true_for_any_databases():
     assert module.use_streaming(
         databases=[flexmock(), flexmock()],
@@ -408,6 +470,98 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port():
     )
 
 
+def test_execute_dump_command_runs_mysqldump_with_tls():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mysqldump',
+            '--defaults-extra-file=/dev/fd/99',
+            '--add-drop-database',
+            '--ssl',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'tls': True},
+            config={},
+            username='root',
+            password='trustsome1',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
+def test_execute_dump_command_runs_mysqldump_without_tls():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mysqldump',
+            '--defaults-extra-file=/dev/fd/99',
+            '--add-drop-database',
+            '--skip-ssl',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={'name': 'foo', 'tls': False},
+            config={},
+            username='root',
+            password='trustsome1',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
 def test_execute_dump_command_runs_mysqldump_with_username_and_password():
     process = flexmock()
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -816,6 +970,90 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
     )
 
 
+def test_restore_data_source_dump_runs_mysql_with_tls():
+    hook_config = [{'name': 'foo', 'tls': True}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args(None, None, None).and_return(())
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mysql',
+            '--batch',
+            '--ssl',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        environment={'USER': 'root'},
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
+def test_restore_data_source_dump_runs_mysql_without_tls():
+    hook_config = [{'name': 'foo', 'tls': False}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module.borgmatic.hooks.credential.parse).should_receive(
+        'resolve_credential'
+    ).replace_with(lambda value, config: value)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'parse_extra_options'
+    ).and_return((), None)
+    flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive(
+        'make_defaults_file_options'
+    ).with_args(None, None, None).and_return(())
+    flexmock(module.os).should_receive('environ').and_return({'USER': 'root'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'mysql',
+            '--batch',
+            '--skip-ssl',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        environment={'USER': 'root'},
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+        borgmatic_runtime_directory='/run/borgmatic',
+    )
+
+
 def test_restore_data_source_dump_runs_mysql_with_username_and_password():
     hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
     extract_process = flexmock(stdout=flexmock())
@@ -922,10 +1160,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
             'password': 'trustsome1',
             'hostname': 'dbhost',
             'port': 'dbport',
+            'tls': True,
             'restore_username': 'restoreuser',
             'restore_password': 'restorepass',
             'restore_hostname': 'restorehost',
             'restore_port': 'restoreport',
+            'restore_tls': False,
         }
     ]
     extract_process = flexmock(stdout=flexmock())
@@ -953,6 +1193,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
             'restoreport',
             '--protocol',
             'tcp',
+            '--skip-ssl',
         ),
         processes=[extract_process],
         output_log_level=logging.DEBUG,