Fix the "spot" check to include borgmatic configuration files that were backed up to support the "bootstrap" action (#1133).

NEWS

@@ -4,6 +4,8 @@
    use.
  * #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
    with orphaned files that need recovery.
+ * #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
+   support the "bootstrap" action.
  * #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
  * When running tests, use Ruff for faster and more comprehensive code linting and formatting,
    replacing Flake8, Black, isort, etc.

borgmatic/actions/check.py

@@ -11,12 +11,14 @@ import shlex
 import shutil
 import textwrap
 
+import borgmatic.actions.config.bootstrap
 import borgmatic.actions.pattern
 import borgmatic.borg.check
 import borgmatic.borg.create
 import borgmatic.borg.environment
 import borgmatic.borg.extract
 import borgmatic.borg.list
+import borgmatic.borg.pattern
 import borgmatic.borg.repo_list
 import borgmatic.borg.state
 import borgmatic.config.paths
@@ -358,11 +360,15 @@ def collect_spot_check_source_paths(
     local_path,
     remote_path,
     borgmatic_runtime_directory,
+    bootstrap_config_paths,
 ):
     '''
     Given a repository configuration dict, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
-    collect the source paths that Borg would use in an actual create (but only include files).
+    arguments as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the
+    bootstrap configuration paths as read from an archive's manifest, collect the source paths that
+    Borg would use in an actual create (but only include files). As part of this, include the
+    bootstrap configuration paths, so that any configuration files included in the archive to
+    support bootstrapping are also spot checked.
     '''
     stream_processes = any(
         borgmatic.hooks.dispatch.call_hooks(
@@ -382,7 +388,14 @@ def collect_spot_check_source_paths(
             list_details=True,
         ),
         patterns=borgmatic.actions.pattern.process_patterns(
-            borgmatic.actions.pattern.collect_patterns(config),
+            borgmatic.actions.pattern.collect_patterns(config)
+            + tuple(
+                borgmatic.borg.pattern.Pattern(
+                    config_path,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                )
+                for config_path in bootstrap_config_paths
+            ),
             config,
             working_directory,
         ),
@@ -609,27 +622,37 @@ def spot_check(
             'The data_tolerance_percentage must be less than or equal to the data_sample_percentage',
         )
 
-    source_paths = collect_spot_check_source_paths(
-        repository,
+    archive = borgmatic.borg.repo_list.resolve_archive_name(
+        repository['path'],
+        'latest',
         config,
         local_borg_version,
         global_arguments,
         local_path,
         remote_path,
-        borgmatic_runtime_directory,
     )
-    logger.debug(f'{len(source_paths)} total source paths for spot check')
+    logger.debug(f'Using archive {archive} for spot check')
 
-    archive = borgmatic.borg.repo_list.resolve_archive_name(
+    bootstrap_config_paths = borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
         repository['path'],
-        'latest',
+        archive,
+        config,
+        local_borg_version,
+        global_arguments,
+        borgmatic_runtime_directory,
+    )
+
+    source_paths = collect_spot_check_source_paths(
+        repository,
         config,
         local_borg_version,
         global_arguments,
         local_path,
         remote_path,
+        borgmatic_runtime_directory,
+        bootstrap_config_paths,
     )
-    logger.debug(f'Using archive {archive} for spot check')
+    logger.debug(f'{len(source_paths)} total source paths for spot check')
 
     archive_paths = collect_spot_check_archive_paths(
         repository,

borgmatic/actions/config/bootstrap.py

@@ -16,67 +16,68 @@ def make_bootstrap_config(bootstrap_arguments):
     Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
     '''
     return {
-        'ssh_command': bootstrap_arguments.ssh_command,
+        'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory,
+        'local_path': bootstrap_arguments.local_path,
+        'remote_path': bootstrap_arguments.remote_path,
         # In case the repo has been moved or is accessed from a different path at the point of
         # bootstrapping.
         'relocated_repo_access_is_ok': True,
+        'ssh_command': bootstrap_arguments.ssh_command,
+        'user_runtime_directory': bootstrap_arguments.user_runtime_directory,
     }
 
 
-def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
+def load_config_paths_from_archive(
+    repository_path,
+    archive_name,
+    config,
+    local_borg_version,
+    global_arguments,
+    borgmatic_runtime_directory,
+):
     '''
-    Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
-    repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
-    borgmatic source directory, destination directory, and whether to strip components), the global
-    arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
-    return the config paths from the manifest.json file in the borgmatic source directory or runtime
-    directory after extracting it from the repository archive.
+    Given a repository path, an archive name, a configuration dict, the local Borg version, the
+    global arguments as an argparse.Namespace, and the borgmatic runtime directory, return the
+    config paths from the manifest.json file in the borgmatic source directory or runtime directory
+    within the repository archive.
 
     Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
     expected configuration path data.
     '''
-    borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
-        {'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory},
-    )
-    config = make_bootstrap_config(bootstrap_arguments)
-
     # Probe for the manifest file in multiple locations, as the default location has moved to the
     # borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
     # still want to support reading the manifest from previously created archives as well.
-    with borgmatic.config.paths.Runtime_directory(
-        {'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
-    ) as borgmatic_runtime_directory:
-        for base_directory in (
-            'borgmatic',
-            borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
-            borgmatic_source_directory,
-        ):
-            borgmatic_manifest_path = 'sh:' + os.path.join(
-                base_directory,
-                'bootstrap',
-                'manifest.json',
-            )
-
-            extract_process = borgmatic.borg.extract.extract_archive(
-                global_arguments.dry_run,
-                bootstrap_arguments.repository,
-                archive_name,
-                [borgmatic_manifest_path],
-                config,
-                local_borg_version,
-                global_arguments,
-                local_path=bootstrap_arguments.local_path,
-                remote_path=bootstrap_arguments.remote_path,
-                extract_to_stdout=True,
-            )
-            manifest_json = extract_process.stdout.read()
-
-            if manifest_json:
-                break
-        else:
-            raise ValueError(
-                'Cannot read configuration paths from archive due to missing bootstrap manifest',
-            )
+    for base_directory in (
+        'borgmatic',
+        borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
+        borgmatic.config.paths.get_borgmatic_source_directory(config),
+    ):
+        borgmatic_manifest_path = 'sh:' + os.path.join(
+            base_directory,
+            'bootstrap',
+            'manifest.json',
+        )
+
+        extract_process = borgmatic.borg.extract.extract_archive(
+            global_arguments.dry_run,
+            repository_path,
+            archive_name,
+            [borgmatic_manifest_path],
+            config,
+            local_borg_version,
+            global_arguments,
+            local_path=config.get('local_path'),
+            remote_path=config.get('remote_path'),
+            extract_to_stdout=True,
+        )
+        manifest_json = extract_process.stdout.read()
+
+        if manifest_json:
+            break
+    else:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to missing archive or bootstrap manifest',
+        )
 
     try:
         manifest_data = json.loads(manifest_json)
@@ -110,12 +111,16 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
         local_path=bootstrap_arguments.local_path,
         remote_path=bootstrap_arguments.remote_path,
     )
-    manifest_config_paths = get_config_paths(
-        archive_name,
-        bootstrap_arguments,
-        global_arguments,
-        local_borg_version,
-    )
+
+    with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
+        manifest_config_paths = load_config_paths_from_archive(
+            bootstrap_arguments.repository,
+            archive_name,
+            config,
+            local_borg_version,
+            global_arguments,
+            borgmatic_runtime_directory,
+        )
 
     logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")