Custom dump command options for MySQL and MariaDB.

Merge pull request #81 from shivansh02/feature/custom-dump-restore-commands-mysql

borgmatic/config/schema.yaml

@@ -971,6 +971,22 @@ properties:
                         a password will only work if MariaDB is configured to
                         trust the configured username without a password.
                     example: trustsome1
+                mariadb_dump_command:
+                    type: string
+                    description: |
+                        Command to use instead of "mariadb-dump". This can 
+                        be used to run a specific mariadb_dump version 
+                        (e.g., one inside a running container). 
+                        Defaults to "mariadb-dump".
+                    example: docker exec mariadb_container mariadb-dump
+                mariadb_command:
+                    type: string
+                    description: |
+                        Command to run instead of "mariadb". This
+                        can be used to run a specific mariadb
+                        version (e.g., one inside a running container). 
+                        Defaults to "mariadb".
+                    example: docker exec mariadb_container mariadb           
                 restore_password:
                     type: string
                     description: |
@@ -1087,6 +1103,21 @@ properties:
                         Password with which to connect to the restore database.
                         Defaults to the "password" option.
                     example: trustsome1
+                mysql_dump_command:
+                    type: string
+                    description: |
+                        Command to use instead of "mysqldump". This can be used
+                        to run a specific mysql_dump version (e.g., one inside 
+                        a running container). Defaults to "mysqldump".
+                    example: docker exec mysql_container mysqldump
+                mysql_command:
+                    type: string
+                    description: |
+                        Command to run instead of "mysql". This
+                        can be used to run a specific mysql
+                        version (e.g., one inside a running container). 
+                        Defaults to "mysql".
+                    example: docker exec mysql_container mysql
                 format:
                     type: string
                     enum: ['sql']

borgmatic/hooks/mariadb.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 import os
+import shlex
 
 from borgmatic.execute import (
     execute_command,
@@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
     if dry_run:
         return ()
 
+    mariadb_show_command = tuple(
+        shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
+    )
     show_command = (
-        ('mariadb',)
+        mariadb_show_command
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
@@ -79,8 +83,12 @@ def execute_dump_command(
         )
         return None
 
+    mariadb_dump_command = tuple(
+        shlex.quote(part)
+        for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
+    )
     dump_command = (
-        ('mariadb-dump',)
+        mariadb_dump_command
         + (tuple(database['options'].split(' ')) if 'options' in database else ())
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
@@ -208,8 +216,12 @@ def restore_data_source_dump(
         'restore_password', data_source.get('password')
     )
 
+    mariadb_restore_command = tuple(
+        shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
+    )
     restore_command = (
-        ('mariadb', '--batch')
+        mariadb_restore_command
+        + ('--batch',)
         + (
             tuple(data_source['restore_options'].split(' '))
             if 'restore_options' in data_source

borgmatic/hooks/mysql.py

@@ -1,6 +1,7 @@
 import copy
 import logging
 import os
+import shlex
 
 from borgmatic.execute import (
     execute_command,
@@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
     if dry_run:
         return ()
 
+    mysql_show_command = tuple(
+        shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql')
+    )
     show_command = (
-        ('mysql',)
+        mysql_show_command
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
@@ -79,8 +83,11 @@ def execute_dump_command(
         )
         return None
 
+    mysql_dump_command = tuple(
+        shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump')
+    )
     dump_command = (
-        ('mysqldump',)
+        mysql_dump_command
         + (tuple(database['options'].split(' ')) if 'options' in database else ())
         + (('--add-drop-database',) if database.get('add_drop_database', True) else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
@@ -207,8 +214,12 @@ def restore_data_source_dump(
         'restore_password', data_source.get('password')
     )
 
+    mysql_restore_command = tuple(
+        shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql')
+    )
     restore_command = (
-        ('mysql', '--batch')
+        mysql_restore_command
+        + ('--batch',)
         + (
             tuple(data_source['restore_options'].split(' '))
             if 'restore_options' in data_source

tests/unit/hooks/test_mariadb.py

@@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mariadb_with_list_options():
     assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
 
 
+def test_database_names_to_dump_runs_non_default_mariadb_with_list_options():
+    database = {
+        'name': 'all',
+        'list_options': '--defaults-extra-file=mariadb.cnf',
+        'mariadb_command': 'custom_mariadb',
+    }
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        extra_environment=None,
+        full_command=(
+            'custom_mariadb',  # Custom MariaDB command
+            '--defaults-extra-file=mariadb.cnf',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+    ).and_return(('foo\nbar')).once()
+
+    assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
+
+
 def test_execute_dump_command_runs_mariadb_dump():
     process = flexmock()
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -315,6 +336,44 @@ def test_execute_dump_command_runs_mariadb_dump_with_options():
     )
 
 
+def test_execute_dump_command_runs_non_default_mariadb_dump_with_options():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'custom_mariadb_dump',  # Custom MariaDB dump command
+            '--stuff=such',
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={
+                'name': 'foo',
+                'mariadb_dump_command': 'custom_mariadb_dump',
+                'options': '--stuff=such',
+            },  # Custom MariaDB dump command specified
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
 def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump():
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
     flexmock(module.os.path).should_receive('exists').and_return(True)
@@ -435,6 +494,36 @@ def test_restore_data_source_dump_runs_mariadb_with_options():
     )
 
 
+def test_restore_data_source_dump_runs_non_default_mariadb_with_options():
+    hook_config = [
+        {'name': 'foo', 'restore_options': '--harder', 'mariadb_command': 'custom_mariadb'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('custom_mariadb', '--batch', '--harder'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
 def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
     hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())

tests/unit/hooks/test_mysql.py

@@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mysql_with_list_options():
     assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
 
 
+def test_database_names_to_dump_runs_non_default_mysql_with_list_options():
+    database = {
+        'name': 'all',
+        'list_options': '--defaults-extra-file=my.cnf',
+        'mysql_command': 'custom_mysql',
+    }
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        extra_environment=None,
+        full_command=(
+            'custom_mysql',  # Custom MySQL command
+            '--defaults-extra-file=my.cnf',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+    ).and_return(('foo\nbar')).once()
+
+    assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
+
+
 def test_execute_dump_command_runs_mysqldump():
     process = flexmock()
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
@@ -315,6 +336,42 @@ def test_execute_dump_command_runs_mysqldump_with_options():
     )
 
 
+def test_execute_dump_command_runs_non_default_mysqldump():
+    process = flexmock()
+    flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'custom_mysqldump',  # Custom MySQL dump command
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            '--result-file',
+            'dump',
+        ),
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.execute_dump_command(
+            database={
+                'name': 'foo',
+                'mysql_dump_command': 'custom_mysqldump',
+            },  # Custom MySQL dump command specified
+            log_prefix='log',
+            dump_path=flexmock(),
+            database_names=('foo',),
+            extra_environment=None,
+            dry_run=False,
+            dry_run_label='',
+        )
+        == process
+    )
+
+
 def test_execute_dump_command_with_duplicate_dump_skips_mysqldump():
     flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
     flexmock(module.os.path).should_receive('exists').and_return(True)
@@ -435,6 +492,34 @@ def test_restore_data_source_dump_runs_mysql_with_options():
     )
 
 
+def test_restore_data_source_dump_runs_non_default_mysql_with_options():
+    hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('custom_mysql', '--batch', '--harder'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+    ).once()
+
+    module.restore_data_source_dump(
+        hook_config,
+        {},
+        'test.yaml',
+        data_source=hook_config[0],
+        dry_run=False,
+        extract_process=extract_process,
+        connection_params={
+            'hostname': None,
+            'port': None,
+            'username': None,
+            'password': None,
+        },
+    )
+
+
 def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
     hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())