Only check archives with matching prefix.

AUTHORS

@@ -8,3 +8,4 @@ newtonne: Read encryption password from external file
 Robin `ypid` Schneider: Support additional options of Borg
 Scott Squires: Custom archive names
 Thomas LÉVEIL: Support for a keep_minutely prune option
+Nick Whyte: Support prefix filtering for archive consistency checks

borgmatic/borg/check.py

@@ -82,10 +82,13 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo
             VERBOSITY_LOTS: ('--debug',),
         }.get(verbosity, ())
 
+        prefix = consistency_config.get('prefix', '{hostname}-')
+        prefix_flags = ('--prefix', prefix) if prefix else ()
+
         full_command = (
             local_path, 'check',
             repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
+        ) + _make_check_flags(checks, check_last) + prefix_flags + remote_path_flags + lock_wait_flags + verbosity_flags
 
         # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
         stdout = None if verbosity_flags else open(os.devnull, 'w')

borgmatic/config/schema.yaml

@@ -218,6 +218,13 @@ map:
                 desc: Restrict the number of checked archives to the last n. Applies only to the
                       "archives" check.
                 example: 3
+            prefix:
+                type: scalar
+                desc: |
+                    When performing consistency checks, only consider archive names starting with
+                    this prefix. Borg placeholders can be used. See the output of
+                    "borg help placeholders" for details. Default is "{hostname}-".
+                example: sourcehostname
     hooks:
         desc: |
             Shell commands or scripts to execute before and after a backup or if an error has occurred.

borgmatic/config/validate.py

@@ -8,6 +8,8 @@ import pykwalify.errors
 from ruamel import yaml
 
 
+logger = logging.getLogger(__name__)
+
 def schema_filename():
     '''
     Path to the installed YAML configuration schema file, used to validate and parse the
@@ -50,6 +52,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
         )
 
+    consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
+    if archive_name_format and not consistency_prefix:
+        logger.warning('Since version 1.2.0, if you provide `archive_name_format`, you must also'
+                       ' specify `consistency.prefix`.')
+
 
 def parse_configuration(config_filename, schema_filename):
     '''

borgmatic/tests/unit/borg/test_check.py

@@ -89,7 +89,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
 )
 def test_check_archives_calls_borg_with_parameters(checks):
     check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
@@ -111,7 +111,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
 def test_check_archives_with_extract_check_calls_extract_only():
     checks = ('extract',)
     check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').never()
     flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
@@ -127,7 +127,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
 
 def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
     checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    consistency_config = {'check_last': None}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -145,7 +145,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
 
 def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
     checks = ('repository',)
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    consistency_config = {'check_last': None}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -162,7 +162,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
 
 
 def test_check_archives_without_any_checks_bails():
-    consistency_config = flexmock().should_receive('get').and_return(None).mock
+    consistency_config = {'check_last': None}
     flexmock(module).should_receive('_parse_checks').and_return(())
     insert_subprocess_never()
 
@@ -177,7 +177,7 @@ def test_check_archives_without_any_checks_bails():
 def test_check_archives_with_local_path_calls_borg_via_local_path():
     checks = ('repository',)
     check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
@@ -200,7 +200,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
 def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
     checks = ('repository',)
     check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
@@ -223,7 +223,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
 def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     checks = ('repository',)
     check_last = flexmock()
-    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
@@ -240,3 +240,26 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
         storage_config={'lock_wait': 5},
         consistency_config=consistency_config,
     )
+
+
+def test_check_archives_with_retention_prefix():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = {'check_last': check_last, 'prefix': 'foo-'}
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--prefix', 'foo-'),
+        stdout=stdout, stderr=STDOUT,
+    )
+
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={},
+        consistency_config=consistency_config,
+    )

borgmatic/tests/unit/config/test_validate.py

@@ -1,4 +1,5 @@
 import pytest
+from flexmock import flexmock
 
 from borgmatic.config import validate as module
 
@@ -23,13 +24,42 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
             },
         )
 
+def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
+    with pytest.raises(module.Validation_error):
+        module.apply_logical_validation(
+            'config.yaml',
+            {
+                'storage': {'archive_name_format': '{hostname}-{now}'},
+                'retention': {'keep_daily': 7},
+                'consistency': {'prefix': '{hostname}-'}
+            },
+        )
+
+
+def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
+    logger = flexmock(module.logger)
+    logger.should_receive('warning').once()
+
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'storage': {'archive_name_format': '{hostname}-{now}'},
+            'retention': {'prefix': '{hostname}-'},
+            'consistency': {},
+        },
+    )
+
+
+def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
+    logger = flexmock(module.logger)
+    logger.should_receive('warning').never()
 
-def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
     module.apply_logical_validation(
         'config.yaml',
         {
             'storage': {'archive_name_format': '{hostname}-{now}'},
             'retention': {'prefix': '{hostname}-'},
+            'consistency': {'prefix': '{hostname}-'}
         },
     )
 

borgmatic/tests/unit/test_verbosity.py

@@ -7,5 +7,5 @@ def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
     assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
 
 
-def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level():
-    assert module.verbosity_to_log_level('my pants') == logging.ERROR
+def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():
+    assert module.verbosity_to_log_level('my pants') == logging.WARNING

borgmatic/verbosity.py

@@ -12,4 +12,4 @@ def verbosity_to_log_level(verbosity):
     return {
         VERBOSITY_SOME: logging.INFO,
         VERBOSITY_LOTS: logging.DEBUG,
-    }.get(verbosity, logging.ERROR)
+    }.get(verbosity, logging.WARNING)