Get automated tests passing (#1143).

borgmatic/actions/check.py

@@ -633,15 +633,6 @@ def spot_check(
     )
     logger.debug(f'Using archive {archive} for spot check')
 
-    bootstrap_config_paths = borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
-        repository['path'],
-        archive,
-        config,
-        local_borg_version,
-        global_arguments,
-        borgmatic_runtime_directory,
-    )
-
     source_paths = collect_spot_check_source_paths(
         repository,
         config,
@@ -650,7 +641,14 @@ def spot_check(
         local_path,
         remote_path,
         borgmatic_runtime_directory,
-        bootstrap_config_paths,
+        bootstrap_config_paths=borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
+            repository['path'],
+            archive,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        ),
     )
     logger.debug(f'{len(source_paths)} total source paths for spot check')
 

tests/unit/actions/config/test_bootstrap.py

@@ -4,38 +4,32 @@ from flexmock import flexmock
 from borgmatic.actions.config import bootstrap as module
 
 
-def test_make_bootstrap_config_uses_ssh_command_argument():
-    ssh_command = flexmock()
+def test_make_bootstrap_config_uses_bootstrap_arguments():
+    config = module.make_bootstrap_config(
+        flexmock(
+            borgmatic_source_directory='/source',
+            local_path='borg1',
+            remote_path='borg2',
+            ssh_command='ssh',
+            user_runtime_directory='/run',
+        )
+    )
 
-    config = module.make_bootstrap_config(flexmock(ssh_command=ssh_command))
-    assert config['ssh_command'] == ssh_command
+    assert config['borgmatic_source_directory'] == '/source'
+    assert config['local_path'] == 'borg1'
+    assert config['remote_path'] == 'borg2'
     assert config['relocated_repo_access_is_ok']
+    assert config['ssh_command'] == 'ssh'
+    assert config['user_runtime_directory'] == '/run'
 
 
-def test_get_config_paths_returns_list_of_config_paths():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/source')
-    flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command=None,
-        local_path='borg7',
-        remote_path='borg8',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
+def test_load_config_paths_from_archive_returns_list_of_config_paths():
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/source')
     extract_process = flexmock(
         stdout=flexmock(
             read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
@@ -45,54 +39,38 @@ def test_get_config_paths_returns_list_of_config_paths():
         extract_process,
     )
 
-    assert module.get_config_paths(
+    assert module.load_config_paths_from_archive(
+        'repo',
         'archive',
-        bootstrap_arguments,
-        global_arguments,
-        local_borg_version,
+        config={},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(dry_run=False),
+        borgmatic_runtime_directory='/run',
     ) == ['/borgmatic/config.yaml']
 
 
-def test_get_config_paths_probes_for_manifest():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/source')
-    flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command=None,
-        local_path='borg7',
-        remote_path='borg8',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    borgmatic_runtime_directory = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        borgmatic_runtime_directory,
-    )
+def test_load_config_paths_from_archive_probes_for_manifest():
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
-    flexmock(module.os.path).should_receive('join').with_args(
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/source')
+    flexmock(module.os.path).should_call('join').with_args(
         'borgmatic',
         'bootstrap',
         'manifest.json',
-    ).and_return('borgmatic/bootstrap/manifest.json').once()
-    flexmock(module.os.path).should_receive('join').with_args(
-        borgmatic_runtime_directory,
+    ).once()
+    flexmock(module.os.path).should_call('join').with_args(
+        '/run',
         'bootstrap',
         'manifest.json',
-    ).and_return('run/borgmatic/bootstrap/manifest.json').once()
-    flexmock(module.os.path).should_receive('join').with_args(
+    ).once()
+    flexmock(module.os.path).should_call('join').with_args(
         '/source',
         'bootstrap',
         'manifest.json',
-    ).and_return('/source/bootstrap/manifest.json').once()
+    ).once()
     manifest_missing_extract_process = flexmock(
         stdout=flexmock(read=lambda: None),
     )
@@ -105,128 +83,46 @@ def test_get_config_paths_probes_for_manifest():
         manifest_missing_extract_process,
     ).and_return(manifest_missing_extract_process).and_return(manifest_found_extract_process)
 
-    assert module.get_config_paths(
+    assert module.load_config_paths_from_archive(
+        'repo',
         'archive',
-        bootstrap_arguments,
-        global_arguments,
-        local_borg_version,
+        config={},
+        local_borg_version=flexmock(),
+        global_arguments=flexmock(dry_run=False),
+        borgmatic_runtime_directory='/run',
     ) == ['/borgmatic/config.yaml']
 
 
-def test_get_config_paths_translates_ssh_command_argument_to_config():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/source')
-    config = {}
-    flexmock(module).should_receive('make_bootstrap_config').and_return(config)
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command='ssh -i key',
-        local_path='borg7',
-        remote_path='borg8',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
+def test_load_config_paths_from_archive_with_missing_manifest_raises_value_error():
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
-    extract_process = flexmock(
-        stdout=flexmock(
-            read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
-        ),
-    )
-    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args(
-        False,
-        'repo',
-        'archive',
-        object,
-        config,
-        object,
-        object,
-        extract_to_stdout=True,
-        local_path='borg7',
-        remote_path='borg8',
-    ).and_return(extract_process)
-
-    assert module.get_config_paths(
-        'archive',
-        bootstrap_arguments,
-        global_arguments,
-        local_borg_version,
-    ) == ['/borgmatic/config.yaml']
-
-
-def test_get_config_paths_with_missing_manifest_raises_value_error():
     flexmock(module.borgmatic.config.paths).should_receive(
         'get_borgmatic_source_directory',
     ).and_return('/source')
-    flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command=None,
-        local_path='borg7',
-        remote_path='borg7',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'make_runtime_directory_glob',
-    ).replace_with(lambda path: path)
-    flexmock(module.os.path).should_receive('join').and_return('run/borgmatic')
     extract_process = flexmock(stdout=flexmock(read=lambda: ''))
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
         extract_process,
     )
 
     with pytest.raises(ValueError):
-        module.get_config_paths(
+        module.load_config_paths_from_archive(
+            'repo',
             'archive',
-            bootstrap_arguments,
-            global_arguments,
-            local_borg_version,
+            config={},
+            local_borg_version=flexmock(),
+            global_arguments=flexmock(dry_run=False),
+            borgmatic_runtime_directory='/run',
         )
 
 
-def test_get_config_paths_with_broken_json_raises_value_error():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/source')
-    flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command=None,
-        local_path='borg7',
-        remote_path='borg7',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
+def test_load_config_paths_from_archive_with_broken_json_raises_value_error():
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/source')
     extract_process = flexmock(
         stdout=flexmock(read=lambda: '{"config_paths": ["/oops'),
     )
@@ -235,38 +131,23 @@ def test_get_config_paths_with_broken_json_raises_value_error():
     )
 
     with pytest.raises(ValueError):
-        module.get_config_paths(
+        module.load_config_paths_from_archive(
+            'repo',
             'archive',
-            bootstrap_arguments,
-            global_arguments,
-            local_borg_version,
+            config={},
+            local_borg_version=flexmock(),
+            global_arguments=flexmock(dry_run=False),
+            borgmatic_runtime_directory='/run',
         )
 
 
-def test_get_config_paths_with_json_missing_key_raises_value_error():
-    flexmock(module.borgmatic.config.paths).should_receive(
-        'get_borgmatic_source_directory',
-    ).and_return('/source')
-    flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    bootstrap_arguments = flexmock(
-        repository='repo',
-        archive='archive',
-        ssh_command=None,
-        local_path='borg7',
-        remote_path='borg7',
-        borgmatic_source_directory=None,
-        user_runtime_directory=None,
-    )
-    global_arguments = flexmock(
-        dry_run=False,
-    )
-    local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
+def test_load_config_paths_from_archive_with_json_missing_key_raises_value_error():
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
+    flexmock(module.borgmatic.config.paths).should_receive(
+        'get_borgmatic_source_directory',
+    ).and_return('/source')
     extract_process = flexmock(
         stdout=flexmock(read=lambda: '{}'),
     )
@@ -275,17 +156,27 @@ def test_get_config_paths_with_json_missing_key_raises_value_error():
     )
 
     with pytest.raises(ValueError):
-        module.get_config_paths(
+        module.load_config_paths_from_archive(
+            'repo',
             'archive',
-            bootstrap_arguments,
-            global_arguments,
-            local_borg_version,
+            config={},
+            local_borg_version=flexmock(),
+            global_arguments=flexmock(dry_run=False),
+            borgmatic_runtime_directory='/run',
         )
 
 
 def test_run_bootstrap_does_not_raise():
     flexmock(module).should_receive('make_bootstrap_config').and_return({})
-    flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml'])
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
+        'archive',
+    )
+    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module).should_receive('load_config_paths_from_archive').and_return(
+        ['/borgmatic/config.yaml']
+    )
     bootstrap_arguments = flexmock(
         repository='repo',
         archive='archive',
@@ -315,9 +206,6 @@ def test_run_bootstrap_does_not_raise():
     flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
         extract_process,
     ).once()
-    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
-        'archive',
-    )
 
     module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)
 
@@ -325,7 +213,21 @@ def test_run_bootstrap_does_not_raise():
 def test_run_bootstrap_translates_ssh_command_argument_to_config():
     config = {}
     flexmock(module).should_receive('make_bootstrap_config').and_return(config)
-    flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml'])
+    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').with_args(
+        'repo',
+        'archive',
+        config,
+        object,
+        object,
+        local_path='borg7',
+        remote_path='borg8',
+    ).and_return('archive')
+    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
+        flexmock(),
+    )
+    flexmock(module).should_receive('load_config_paths_from_archive').and_return(
+        ['/borgmatic/config.yaml']
+    )
     bootstrap_arguments = flexmock(
         repository='repo',
         archive='archive',
@@ -341,9 +243,6 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config():
         dry_run=False,
     )
     local_borg_version = flexmock()
-    flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return(
-        flexmock(),
-    )
     flexmock(module.borgmatic.config.paths).should_receive(
         'make_runtime_directory_glob',
     ).replace_with(lambda path: path)
@@ -366,14 +265,5 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config():
         local_path='borg7',
         remote_path='borg8',
     ).and_return(extract_process).once()
-    flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').with_args(
-        'repo',
-        'archive',
-        config,
-        object,
-        object,
-        local_path='borg7',
-        remote_path='borg8',
-    ).and_return('archive')
 
     module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)

tests/unit/actions/test_check.py

@@ -580,7 +580,7 @@ def test_upgrade_check_times_renames_stale_temporary_check_path():
     module.upgrade_check_times(flexmock(), flexmock())
 
 
-def test_collect_spot_check_source_paths_parses_borg_output():
+def test_collect_spot_check_source_paths_parses_borg_output_and_includes_bootstrap_config_paths():
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return(
         {'hook1': False, 'hook2': True},
     )
@@ -588,9 +588,16 @@ def test_collect_spot_check_source_paths_parses_borg_output():
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
-    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
+    flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').with_args(
+        (
+            Pattern('collected', source=module.borgmatic.borg.pattern.Pattern_source.HOOK),
+            Pattern('extra.yaml', source=module.borgmatic.borg.pattern.Pattern_source.INTERNAL),
+        ),
+        config=object,
+        working_directory=None,
+    ).and_return(
         [Pattern('foo'), Pattern('bar')],
     )
     flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args(
@@ -624,6 +631,7 @@ def test_collect_spot_check_source_paths_parses_borg_output():
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=('extra.yaml',),
     ) == ('/etc/path', '/etc/other')
 
 
@@ -635,7 +643,7 @@ def test_collect_spot_check_source_paths_omits_progress_from_create_dry_run_comm
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')],
@@ -671,6 +679,7 @@ def test_collect_spot_check_source_paths_omits_progress_from_create_dry_run_comm
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=(),
     ) == ('/etc/path', '/etc/other')
 
 
@@ -682,7 +691,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')],
@@ -718,6 +727,7 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=(),
     ) == ('/etc/path', '/etc/other')
 
 
@@ -729,7 +739,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')],
@@ -765,6 +775,7 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=(),
     ) == ('/etc/path', '/etc/other')
 
 
@@ -776,7 +787,7 @@ def test_collect_spot_check_source_paths_skips_directories():
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')],
@@ -814,6 +825,7 @@ def test_collect_spot_check_source_paths_skips_directories():
             local_path=flexmock(),
             remote_path=flexmock(),
             borgmatic_runtime_directory='/run/borgmatic',
+            bootstrap_config_paths=(),
         )
         == ()
     )
@@ -921,7 +933,7 @@ def test_collect_spot_check_source_paths_uses_working_directory():
         flexmock(),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('collect_patterns').and_return(
-        flexmock(),
+        (Pattern('collected'),),
     )
     flexmock(module.borgmatic.actions.pattern).should_receive('process_patterns').and_return(
         [Pattern('foo'), Pattern('bar')],
@@ -960,6 +972,7 @@ def test_collect_spot_check_source_paths_uses_working_directory():
         local_path=flexmock(),
         remote_path=flexmock(),
         borgmatic_runtime_directory='/run/borgmatic',
+        bootstrap_config_paths=(),
     ) == ('foo', 'bar')
 
 
@@ -1384,6 +1397,9 @@ def test_spot_check_with_count_delta_greater_than_count_tolerance_percentage_err
     flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
         'archive',
     )
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive(
+        'load_config_paths_from_archive'
+    ).and_return(('bootstrap.yaml',))
     flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(
         ('/foo', '/bar'),
     ).once()
@@ -1416,6 +1432,9 @@ def test_spot_check_with_failing_percentage_greater_than_data_tolerance_percenta
     flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
         'archive',
     )
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive(
+        'load_config_paths_from_archive'
+    ).and_return(('bootstrap.yaml',))
     flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
     flexmock(module).should_receive('compare_spot_check_hashes').and_return(
         ('/bar', '/baz', '/quux'),
@@ -1449,6 +1468,9 @@ def test_spot_check_with_high_enough_tolerances_does_not_raise():
     flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
         'archive',
     )
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive(
+        'load_config_paths_from_archive'
+    ).and_return(('bootstrap.yaml',))
     flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
     flexmock(module).should_receive('compare_spot_check_hashes').and_return(
         ('/bar', '/baz', '/quux'),
@@ -1479,6 +1501,9 @@ def test_spot_check_without_any_source_paths_errors():
     flexmock(module.borgmatic.borg.repo_list).should_receive('resolve_archive_name').and_return(
         'archive',
     )
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive(
+        'load_config_paths_from_archive'
+    ).and_return(('bootstrap.yaml',))
     flexmock(module).should_receive('collect_spot_check_archive_paths').and_return(('/foo', '/bar'))
     flexmock(module).should_receive('compare_spot_check_hashes').never()