Get tests passing (#261).

borgmatic/actions/check.py

@@ -373,12 +373,10 @@ def collect_spot_check_source_paths(
             dry_run=True,
             repository_path=repository['path'],
             config=config,
-            config_paths=(),
-            source_directories=borgmatic.actions.create.process_source_directories(
-                config, (), borgmatic_runtime_directory
-            ),
+            source_directories=borgmatic.actions.create.process_source_directories(config, ()),
             local_borg_version=local_borg_version,
             global_arguments=global_arguments,
+            borgmatic_runtime_directory=borgmatic_runtime_directory,
             local_path=local_path,
             remote_path=remote_path,
             list_files=True,

borgmatic/actions/create.py

@@ -129,6 +129,9 @@ def deduplicate_directories(directory_devices, additional_directory_devices):
     return sorted(deduplicated)
 
 
+ROOT_PATTERN_PREFIX = 'R '
+
+
 def pattern_root_directories(patterns=None):
     '''
     Given a sequence of patterns, parse out and return just the root directories.
@@ -239,7 +242,6 @@ def run_create(
             global_arguments.dry_run,
             repository['path'],
             config,
-            config_paths,
             source_directories,
             local_borg_version,
             global_arguments,

borgmatic/borg/create.py

@@ -134,9 +134,6 @@ def make_list_filter_flags(local_borg_version, dry_run):
         return f'{base_flags}-'
 
 
-ROOT_PATTERN_PREFIX = 'R '
-
-
 def special_file(path):
     '''
     Return whether the given path is a special file (character device, block device, or named pipe
@@ -197,21 +194,15 @@ def collect_special_file_paths(
     )
 
 
-def check_all_source_directories_exist(source_directories, working_directory=None):
+def check_all_source_directories_exist(source_directories):
     '''
-    Given a sequence of source directories and an optional working directory to serve as a prefix
-    for each (if it's a relative directory), check that the source directories all exist. If any do
+    Given a sequence of source directories, check that the source directories all exist. If any do
     not, raise an exception.
     '''
     missing_directories = [
         source_directory
         for source_directory in source_directories
-        if not all(
-            [
-                os.path.exists(os.path.join(working_directory or '', directory))
-                for directory in expand_directory(source_directory, working_directory)
-            ]
-        )
+        if not os.path.exists(source_directory)
     ]
     if missing_directories:
         raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
@@ -224,10 +215,10 @@ def make_base_create_command(
     dry_run,
     repository_path,
     config,
-    config_paths,
     source_directories,
     local_borg_version,
     global_arguments,
+    borgmatic_runtime_directory,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -242,12 +233,8 @@ def make_base_create_command(
     (base Borg create command flags, Borg create command positional arguments, open pattern file
     handle, open exclude file handle).
     '''
-    working_directory = borgmatic.config.paths.get_working_directory(config)
-
     if config.get('source_directories_must_exist', False):
-        check_all_source_directories_exist(
-            config.get('source_directories'), working_directory=working_directory
-        )
+        check_all_source_directories_exist(source_directories)
 
     ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
 
@@ -339,6 +326,7 @@ def make_base_create_command(
             f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
         borg_environment = environment.make_environment(config)
+        working_directory = borgmatic.config.paths.get_working_directory(config)
 
         logger.debug(f'{repository_path}: Collecting special file paths')
         special_file_paths = collect_special_file_paths(
@@ -376,7 +364,6 @@ def create_archive(
     dry_run,
     repository_path,
     config,
-    config_paths,
     source_directories,
     local_borg_version,
     global_arguments,
@@ -406,10 +393,10 @@ def create_archive(
             dry_run,
             repository_path,
             config,
-            config_paths,
             source_directories,
             local_borg_version,
             global_arguments,
+            borgmatic_runtime_directory,
             local_path,
             remote_path,
             progress,

borgmatic/config/generate.py

@@ -44,12 +44,12 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     if example is not None:
         return example
 
-    if schema_type == 'array' or 'array' in schema_type:
+    if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type):
         config = ruamel.yaml.comments.CommentedSeq(
             [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
         )
         add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
-    elif schema_type == 'object' or 'object' in schema_type:
+    elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type):
         config = ruamel.yaml.comments.CommentedMap(
             [
                 (field_name, schema_to_sample_configuration(sub_schema, level + 1))

borgmatic/hooks/mongodb.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import shlex
 
 import borgmatic.config.paths

borgmatic/hooks/zfs.py

@@ -1,7 +1,6 @@
 import glob
 import logging
 import os
-import shlex
 import shutil
 import subprocess
 
@@ -66,7 +65,7 @@ def get_all_datasets(zfs_command):
             '-t',
             'filesystem',
             '-o',
-            f'name,mountpoint',
+            'name,mountpoint',
         )
     )
 

tests/unit/actions/test_check.py

@@ -557,14 +557,17 @@ def test_collect_spot_check_source_paths_parses_borg_output():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
+    flexmock(module.borgmatic.actions.create).should_receive(
+        'process_source_directories'
+    ).and_return(['foo', 'bar'])
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        config_paths=(),
+        source_directories=['foo', 'bar'],
         local_borg_version=object,
         global_arguments=object,
-        borgmatic_runtime_directories=(),
+        borgmatic_runtime_directory='/run/borgmatic',
         local_path=object,
         remote_path=object,
         list_files=True,
@@ -588,6 +591,7 @@ def test_collect_spot_check_source_paths_parses_borg_output():
         global_arguments=flexmock(),
         local_path=flexmock(),
         remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/etc/path', '/etc/other')
 
 
@@ -595,14 +599,17 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': False}
     )
+    flexmock(module.borgmatic.actions.create).should_receive(
+        'process_source_directories'
+    ).and_return(['foo', 'bar'])
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        config_paths=(),
+        source_directories=['foo', 'bar'],
         local_borg_version=object,
         global_arguments=object,
-        borgmatic_runtime_directories=(),
+        borgmatic_runtime_directory='/run/borgmatic',
         local_path=object,
         remote_path=object,
         list_files=True,
@@ -626,6 +633,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
         global_arguments=flexmock(),
         local_path=flexmock(),
         remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/etc/path', '/etc/other')
 
 
@@ -633,14 +641,17 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
+    flexmock(module.borgmatic.actions.create).should_receive(
+        'process_source_directories'
+    ).and_return(['foo', 'bar'])
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        config_paths=(),
+        source_directories=['foo', 'bar'],
         local_borg_version=object,
         global_arguments=object,
-        borgmatic_runtime_directories=(),
+        borgmatic_runtime_directory='/run/borgmatic',
         local_path=object,
         remote_path=object,
         list_files=True,
@@ -664,6 +675,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
         global_arguments=flexmock(),
         local_path=flexmock(),
         remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('/etc/path', '/etc/other')
 
 
@@ -671,14 +683,17 @@ def test_collect_spot_check_source_paths_skips_directories():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
+    flexmock(module.borgmatic.actions.create).should_receive(
+        'process_source_directories'
+    ).and_return(['foo', 'bar'])
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        config_paths=(),
+        source_directories=['foo', 'bar'],
         local_borg_version=object,
         global_arguments=object,
-        borgmatic_runtime_directories=(),
+        borgmatic_runtime_directory='/run/borgmatic',
         local_path=object,
         remote_path=object,
         list_files=True,
@@ -704,6 +719,7 @@ def test_collect_spot_check_source_paths_skips_directories():
             global_arguments=flexmock(),
             local_path=flexmock(),
             remote_path=flexmock(),
+            borgmatic_runtime_directory='/run/borgmatic',
         )
         == ()
     )
@@ -806,14 +822,17 @@ def test_collect_spot_check_source_paths_uses_working_directory():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True}
     )
+    flexmock(module.borgmatic.actions.create).should_receive(
+        'process_source_directories'
+    ).and_return(['foo', 'bar'])
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
         dry_run=True,
         repository_path='repo',
         config=object,
-        config_paths=(),
+        source_directories=['foo', 'bar'],
         local_borg_version=object,
         global_arguments=object,
-        borgmatic_runtime_directories=(),
+        borgmatic_runtime_directory='/run/borgmatic',
         local_path=object,
         remote_path=object,
         list_files=True,
@@ -840,6 +859,7 @@ def test_collect_spot_check_source_paths_uses_working_directory():
         global_arguments=flexmock(),
         local_path=flexmock(),
         remote_path=flexmock(),
+        borgmatic_runtime_directory='/run/borgmatic',
     ) == ('foo', 'bar')
 
 

tests/unit/actions/test_create.py

@@ -1,10 +1,221 @@
 import sys
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
 
 
+def test_create_borgmatic_manifest_creates_manifest_file():
+    flexmock(module.os.path).should_receive('join').with_args(
+        '/run/borgmatic', 'bootstrap', 'manifest.json'
+    ).and_return('/run/borgmatic/bootstrap/manifest.json')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
+    flexmock(sys.modules['builtins']).should_receive('open').with_args(
+        '/run/borgmatic/bootstrap/manifest.json', 'w'
+    ).and_return(
+        flexmock(
+            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
+            __exit__=lambda *args: None,
+        )
+    )
+    flexmock(module.json).should_receive('dump').and_return(True).once()
+
+    module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', False)
+
+
+def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_runtime_directory():
+    flexmock(module.os.path).should_receive('join').with_args(
+        '/run/borgmatic', 'bootstrap', 'manifest.json'
+    ).and_return('/run/borgmatic/bootstrap/manifest.json')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
+    flexmock(sys.modules['builtins']).should_receive('open').with_args(
+        '/run/borgmatic/bootstrap/manifest.json', 'w'
+    ).and_return(
+        flexmock(
+            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
+            __exit__=lambda *args: None,
+        )
+    )
+    flexmock(module.json).should_receive('dump').and_return(True).once()
+
+    module.create_borgmatic_manifest(
+        {'borgmatic_runtime_directory': '/borgmatic'}, 'test.yaml', '/run/borgmatic', False
+    )
+
+
+def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run():
+    flexmock(module.json).should_receive('dump').never()
+
+    module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', True)
+
+
+def test_expand_directory_with_basic_path_passes_it_through():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').and_return([])
+
+    paths = module.expand_directory('foo', None)
+
+    assert paths == ['foo']
+
+
+def test_expand_directory_with_glob_expands():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').and_return(['foo', 'food'])
+
+    paths = module.expand_directory('foo*', None)
+
+    assert paths == ['foo', 'food']
+
+
+def test_expand_directory_with_working_directory_passes_it_through():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo').and_return([]).once()
+
+    paths = module.expand_directory('foo', working_directory='/working/dir')
+
+    assert paths == ['/working/dir/foo']
+
+
+def test_expand_directory_with_glob_passes_through_working_directory():
+    flexmock(module.os.path).should_receive('expanduser').and_return('foo*')
+    flexmock(module.glob).should_receive('glob').with_args('/working/dir/foo*').and_return(
+        ['/working/dir/foo', '/working/dir/food']
+    ).once()
+
+    paths = module.expand_directory('foo*', working_directory='/working/dir')
+
+    assert paths == ['/working/dir/foo', '/working/dir/food']
+
+
+def test_expand_directories_flattens_expanded_directories():
+    flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return(
+        ['/root/foo']
+    )
+    flexmock(module).should_receive('expand_directory').with_args('bar*', None).and_return(
+        ['bar', 'barf']
+    )
+
+    paths = module.expand_directories(('~/foo', 'bar*'))
+
+    assert paths == ('/root/foo', 'bar', 'barf')
+
+
+def test_expand_directories_with_working_directory_passes_it_through():
+    flexmock(module).should_receive('expand_directory').with_args('foo', '/working/dir').and_return(
+        ['/working/dir/foo']
+    )
+
+    paths = module.expand_directories(('foo',), working_directory='/working/dir')
+
+    assert paths == ('/working/dir/foo',)
+
+
+def test_expand_directories_considers_none_as_no_directories():
+    paths = module.expand_directories(None, None)
+
+    assert paths == ()
+
+
+def test_map_directories_to_devices_gives_device_id_per_path():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66))
+
+    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+
+    assert device_map == {
+        '/foo': 55,
+        '/bar': 66,
+    }
+
+
+def test_map_directories_to_devices_with_missing_path_does_not_error():
+    flexmock(module.os.path).should_receive('exists').and_return(True).and_return(False)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').never()
+
+    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+
+    assert device_map == {
+        '/foo': 55,
+        '/bar': None,
+    }
+
+
+def test_map_directories_to_devices_uses_working_directory_to_construct_path():
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/working/dir/bar').and_return(
+        flexmock(st_dev=66)
+    )
+
+    device_map = module.map_directories_to_devices(
+        ('/foo', 'bar'), working_directory='/working/dir'
+    )
+
+    assert device_map == {
+        '/foo': 55,
+        'bar': 66,
+    }
+
+
+@pytest.mark.parametrize(
+    'directories,additional_directories,expected_directories',
+    (
+        ({'/': 1, '/root': 1}, {}, ['/']),
+        ({'/': 1, '/root/': 1}, {}, ['/']),
+        ({'/': 1, '/root': 2}, {}, ['/', '/root']),
+        ({'/root': 1, '/': 1}, {}, ['/']),
+        ({'/root': 1, '/root/foo': 1}, {}, ['/root']),
+        ({'/root/': 1, '/root/foo': 1}, {}, ['/root/']),
+        ({'/root': 1, '/root/foo/': 1}, {}, ['/root']),
+        ({'/root': 1, '/root/foo': 2}, {}, ['/root', '/root/foo']),
+        ({'/root/foo': 1, '/root': 1}, {}, ['/root']),
+        ({'/root': None, '/root/foo': None}, {}, ['/root', '/root/foo']),
+        ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, {}, ['/etc', '/root']),
+        ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, {}, ['/root']),
+        ({'/dup': 1, '/dup': 1}, {}, ['/dup']),
+        ({'/foo': 1, '/bar': 1}, {}, ['/bar', '/foo']),
+        ({'/foo': 1, '/bar': 2}, {}, ['/bar', '/foo']),
+        ({'/root/foo': 1}, {'/root': 1}, []),
+        ({'/root/foo': 1}, {'/root': 2}, ['/root/foo']),
+        ({'/root/foo': 1}, {}, ['/root/foo']),
+    ),
+)
+def test_deduplicate_directories_removes_child_paths_on_the_same_filesystem(
+    directories, additional_directories, expected_directories
+):
+    assert (
+        module.deduplicate_directories(directories, additional_directories) == expected_directories
+    )
+
+
+def test_pattern_root_directories_deals_with_none_patterns():
+    assert module.pattern_root_directories(patterns=None) == []
+
+
+def test_pattern_root_directories_parses_roots_and_ignores_others():
+    assert module.pattern_root_directories(
+        ['R /root', '+ /root/foo', '- /root/foo/bar', 'R /baz']
+    ) == ['/root', '/baz']
+
+
+# TODO
+# def test_process_source_directories_...
+#    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+#    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+#    flexmock(module).should_receive('expand_directories').and_return(())
+#    flexmock(module).should_receive('pattern_root_directories').and_return([])
+#    ...
+
+
 def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
@@ -18,6 +229,8 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
+    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=None,
         progress=flexmock(),
@@ -57,6 +270,8 @@ def test_run_create_with_store_config_files_false_does_not_create_borgmatic_mani
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
+    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=None,
         progress=flexmock(),
@@ -98,6 +313,8 @@ def test_run_create_runs_with_selected_repository():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
+    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -177,6 +394,8 @@ def test_run_create_produces_json():
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
         'call_hooks_even_if_unconfigured'
     ).and_return({})
+    flexmock(module).should_receive('process_source_directories').and_return([])
+    flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap')
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -201,53 +420,3 @@ def test_run_create_produces_json():
             remote_path=None,
         )
     ) == [parsed_json]
-
-
-def test_create_borgmatic_manifest_creates_manifest_file():
-    flexmock(module.os.path).should_receive('join').with_args(
-        '/run/borgmatic', 'bootstrap', 'manifest.json'
-    ).and_return('/run/borgmatic/bootstrap/manifest.json')
-    flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(module.os).should_receive('makedirs').and_return(True)
-
-    flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(
-        '/run/borgmatic/bootstrap/manifest.json', 'w'
-    ).and_return(
-        flexmock(
-            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
-            __exit__=lambda *args: None,
-        )
-    )
-    flexmock(module.json).should_receive('dump').and_return(True).once()
-
-    module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', False)
-
-
-def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_runtime_directory():
-    flexmock(module.os.path).should_receive('join').with_args(
-        '/run/borgmatic', 'bootstrap', 'manifest.json'
-    ).and_return('/run/borgmatic/bootstrap/manifest.json')
-    flexmock(module.os.path).should_receive('exists').and_return(False)
-    flexmock(module.os).should_receive('makedirs').and_return(True)
-
-    flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(
-        '/run/borgmatic/bootstrap/manifest.json', 'w'
-    ).and_return(
-        flexmock(
-            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
-            __exit__=lambda *args: None,
-        )
-    )
-    flexmock(module.json).should_receive('dump').and_return(True).once()
-
-    module.create_borgmatic_manifest(
-        {'borgmatic_runtime_directory': '/borgmatic'}, 'test.yaml', '/run/borgmatic', False
-    )
-
-
-def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run():
-    flexmock(module.json).should_receive('dump').never()
-
-    module.create_borgmatic_manifest({}, 'test.yaml', '/run/borgmatic', True)

tests/unit/config/test_generate.py

@@ -103,6 +103,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e
     assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
 
 
+def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_multiple_types():
+    schema = {
+        'type': 'array',
+        'items': {
+            'type': ['object', 'null'],
+            'properties': OrderedDict(
+                [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
+            ),
+        },
+    }
+    flexmock(module).should_receive('get_properties').and_return(schema['items']['properties'])
+    flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list)
+    flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    flexmock(module).should_receive('add_comments_to_configuration_object')
+
+    config = module.schema_to_sample_configuration(schema)
+
+    assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
+
+
 def test_schema_to_sample_configuration_with_unsupported_schema_raises():
     schema = {'gobbledygook': [{'type': 'not-your'}]}
 

tests/unit/hooks/test_dispatch.py

@@ -77,10 +77,12 @@ def test_call_hooks_calls_skips_return_values_for_missing_hooks():
     assert return_values == expected_return_values
 
 
-def test_call_hooks_calls_skips_return_values_for_null_hooks():
+def test_call_hooks_calls_treats_null_hook_as_optionless():
     config = {'super_hook': flexmock(), 'other_hook': None}
-    expected_return_values = {'super_hook': flexmock()}
-    flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook'])
+    expected_return_values = {'super_hook': flexmock(), 'other_hook': flexmock()}
+    flexmock(module).should_receive('call_hook').and_return(
+        expected_return_values['super_hook']
+    ).and_return(expected_return_values['other_hook'])
 
     return_values = module.call_hooks(
         'do_stuff', config, 'prefix', ('super_hook', 'other_hook'), 55

tests/unit/hooks/test_mariadb.py

@@ -75,7 +75,12 @@ def test_dump_data_sources_dumps_each_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -100,7 +105,12 @@ def test_dump_data_sources_dumps_with_password():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        [database], {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        [database],
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -120,7 +130,12 @@ def test_dump_data_sources_dumps_all_databases_at_once():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -143,7 +158,12 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -449,7 +469,12 @@ def test_dump_data_sources_errors_for_missing_all_databases():
 
     with pytest.raises(ValueError):
         assert module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
 
 
@@ -463,7 +488,12 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=True,
         )
         == []
     )

tests/unit/hooks/test_mongodb.py

@@ -43,7 +43,12 @@ def test_dump_data_sources_runs_mongodump_for_each_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -60,7 +65,12 @@ def test_dump_data_sources_with_dry_run_skips_mongodump():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=True,
         )
         == []
     )
@@ -93,7 +103,12 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -133,7 +148,12 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -153,7 +173,12 @@ def test_dump_data_sources_runs_mongodump_with_directory_format():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == []
     )
@@ -183,7 +208,12 @@ def test_dump_data_sources_runs_mongodump_with_options():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -203,7 +233,12 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 

tests/unit/hooks/test_mysql.py

@@ -75,7 +75,12 @@ def test_dump_data_sources_dumps_each_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -100,7 +105,12 @@ def test_dump_data_sources_dumps_with_password():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        [database], {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        [database],
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -120,7 +130,12 @@ def test_dump_data_sources_dumps_all_databases_at_once():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -143,7 +158,12 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -447,7 +467,12 @@ def test_dump_data_sources_errors_for_missing_all_databases():
 
     with pytest.raises(ValueError):
         assert module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
 
 
@@ -461,7 +486,12 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=True,
         )
         == []
     )

tests/unit/hooks/test_postgresql.py

@@ -253,7 +253,12 @@ def test_dump_data_sources_runs_pg_dump_for_each_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -267,7 +272,12 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump():
 
     with pytest.raises(ValueError):
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
 
 
@@ -278,7 +288,12 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump():
     flexmock(module).should_receive('database_names_to_dump').and_return(())
 
     module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=True,
     ) == []
 
 
@@ -298,7 +313,12 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == []
     )
@@ -320,7 +340,12 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=True,
         )
         == []
     )
@@ -360,7 +385,12 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -398,7 +428,12 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -436,7 +471,12 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -470,7 +510,12 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == []
     )
@@ -507,7 +552,12 @@ def test_dump_data_sources_runs_pg_dump_with_options():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -531,7 +581,12 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 
@@ -567,7 +622,12 @@ def test_dump_data_sources_runs_non_default_pg_dump():
     ).and_return(process).once()
 
     assert module.dump_data_sources(
-        databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+        databases,
+        {},
+        'test.yaml',
+        borgmatic_runtime_directory='/run/borgmatic',
+        source_directories=[],
+        dry_run=False,
     ) == [process]
 
 

tests/unit/hooks/test_sqlite.py

@@ -28,7 +28,12 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == []
     )
@@ -53,7 +58,12 @@ def test_dump_data_sources_dumps_each_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -85,7 +95,12 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -108,7 +123,12 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -133,7 +153,12 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=False
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=False,
         )
         == processes
     )
@@ -152,7 +177,12 @@ def test_dump_data_sources_does_not_dump_if_dry_run():
 
     assert (
         module.dump_data_sources(
-            databases, {}, 'test.yaml', borgmatic_runtime_directory='/run/borgmatic', dry_run=True
+            databases,
+            {},
+            'test.yaml',
+            borgmatic_runtime_directory='/run/borgmatic',
+            source_directories=[],
+            dry_run=True,
         )
         == []
     )