Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56).

NEWS

@@ -1,7 +1,9 @@
-1.1.15.dev0
+1.1.15
  * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
  * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
- * #55: Fix for missing tags/releases from Gitea and GitHub project hosting.
+   Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
+ * #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
+ * #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
  * #58: Support for using tilde in exclude_patterns to reference home directory.
 
 1.1.14

borgmatic/borg/check.py

@@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None):
     ) + last_flag
 
 
-def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None):
+def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
+                   remote_path=None):
     '''
-    Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
-    local/remote commands to run, check the contained Borg archives for consistency.
+    Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
+    config dict, and a local/remote commands to run, check the contained Borg archives for
+    consistency.
 
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
     check_last = consistency_config.get('check_last', None)
+    lock_wait = None
 
     if set(checks).intersection(set(DEFAULT_CHECKS)):
         remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+        lock_wait = storage_config.get('lock_wait', None)
+        lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
         verbosity_flags = {
             VERBOSITY_SOME: ('--info',),
             VERBOSITY_LOTS: ('--debug',),
@@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
         full_command = (
             local_path, 'check',
             repository,
-        ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
+        ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
 
         # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
         stdout = None if verbosity_flags else open(os.devnull, 'w')
@@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
         subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
 
     if 'extract' in checks:
-        extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path)
+        extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -129,6 +129,8 @@ def create_archive(
     remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
+    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
     files_cache = location_config.get('files_cache')
     files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
@@ -149,7 +151,7 @@ def create_archive(
         ),
     ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
         one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
-        verbosity_flags + dry_run_flags
+        lock_wait_flags + verbosity_flags + dry_run_flags
 
     logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 logger = logging.getLogger(__name__)
 
 
-def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None):
+def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
     '''
     Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
     the dry-run.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info',),
         VERBOSITY_LOTS: ('--debug',),
@@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
         local_path, 'list',
         '--short',
         repository,
-    ) + remote_path_flags + verbosity_flags
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags
 
     list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
 
@@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
             repository=repository,
             last_archive_name=last_archive_name,
         ),
-    ) + remote_path_flags + verbosity_flags + list_flag
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
 
     logger.debug(' '.join(full_extract_command))
     subprocess.check_call(full_extract_command)

borgmatic/borg/prune.py

@@ -32,12 +32,16 @@ def _make_prune_flags(retention_config):
     )
 
 
-def prune_archives(verbosity, dry_run, repository, retention_config, local_path='borg', remote_path=None):
+def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
+                   local_path='borg', remote_path=None):
     '''
-    Given verbosity/dry-run flags, a local or remote repository path, a retention config dict, prune
-    Borg archives according the the retention policy specified in that configuration.
+    Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
+    retention config dict, prune Borg archives according the the retention policy specified in that
+    configuration.
     '''
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
+    lock_wait = storage_config.get('lock_wait', None)
+    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info', '--stats',),
         VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
@@ -51,7 +55,7 @@ def prune_archives(verbosity, dry_run, repository, retention_config, local_path=
         element
         for pair in _make_prune_flags(retention_config)
         for element in pair
-    ) + remote_path_flags + verbosity_flags + dry_run_flags
+    ) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags
 
     logger.debug(' '.join(full_command))
     subprocess.check_call(full_command)

borgmatic/commands/borgmatic.py

@@ -113,6 +113,7 @@ def run_configuration(config_filename, args):  # pragma: no cover
                     args.verbosity,
                     args.dry_run,
                     repository,
+                    storage,
                     retention,
                     local_path=local_path,
                     remote_path=remote_path,
@@ -133,6 +134,7 @@ def run_configuration(config_filename, args):  # pragma: no cover
                 check.check_archives(
                     args.verbosity,
                     repository,
+                    storage,
                     consistency,
                     local_path=local_path,
                     remote_path=remote_path,

borgmatic/config/schema.yaml

@@ -139,6 +139,10 @@ map:
                 type: scalar
                 desc: Umask to be used for borg create.
                 example: 0077
+            lock_wait:
+                type: int
+                desc: Maximum seconds to wait for acquiring a repository/cache lock.
+                example: 5
             archive_name_format:
                 type: scalar
                 desc: |
@@ -152,6 +156,7 @@ map:
         desc: |
             Retention policy for how many backups to keep in each category. See
             https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            At least one of the "keep" options is required for pruning to work.
         map:
             keep_within:
                 type: scalar

borgmatic/tests/unit/borg/test_check.py

@@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
     module.check_archives(
         verbosity=VERBOSITY_SOME,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
     module.check_archives(
         verbosity=VERBOSITY_LOTS,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
     )
 
@@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
         local_path='borg1',
     )
@@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     module.check_archives(
         verbosity=None,
         repository='repo',
+        storage_config={},
         consistency_config=consistency_config,
         remote_path='borg1',
     )
+
+
+def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    checks = ('repository',)
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
+    stdout = flexmock()
+    insert_subprocess_mock(
+        ('borg', 'check', 'repo', '--lock-wait', '5'),
+        stdout=stdout, stderr=STDOUT,
+    )
+    flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
+    flexmock(module.os).should_receive('devnull')
+
+    module.check_archives(
+        verbosity=None,
+        repository='repo',
+        storage_config={'lock_wait': 5},
+        consistency_config=consistency_config,
+    )

borgmatic/tests/unit/borg/test_create.py

@@ -518,6 +518,26 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     )
 
 
+def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+
+    module.create_archive(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'lock_wait': 5},
+    )
+
+
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)

borgmatic/tests/unit/borg/test_extract.py

@@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
     module.extract_last_archive_dry_run(
         verbosity=VERBOSITY_SOME,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
     module.extract_last_archive_dry_run(
         verbosity=VERBOSITY_LOTS,
         repository='repo',
+        lock_wait=None,
     )
 
 
@@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
         local_path='borg1',
     )
 
@@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
     module.extract_last_archive_dry_run(
         verbosity=None,
         repository='repo',
+        lock_wait=None,
         remote_path='borg1',
     )
+
+
+def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
+    flexmock(sys.stdout).encoding = 'utf-8'
+    insert_subprocess_check_output_mock(
+        ('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
+        result='archive1\narchive2\n'.encode('utf-8'),
+    )
+    insert_subprocess_mock(
+        ('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
+    )
+
+    module.extract_last_archive_dry_run(
+        verbosity=None,
+        repository='repo',
+        lock_wait=5,
+    )

borgmatic/tests/unit/borg/test_prune.py

@@ -66,6 +66,7 @@ def test_prune_archives_calls_borg_with_parameters():
         verbosity=None,
         dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
     )
 
@@ -79,6 +80,7 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
 
     module.prune_archives(
         repository='repo',
+        storage_config={},
         verbosity=VERBOSITY_SOME,
         dry_run=False,
         retention_config=retention_config,
@@ -94,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
 
     module.prune_archives(
         repository='repo',
+        storage_config={},
         verbosity=VERBOSITY_LOTS,
         dry_run=False,
         retention_config=retention_config,
@@ -109,6 +112,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
 
     module.prune_archives(
         repository='repo',
+        storage_config={},
         verbosity=None,
         dry_run=True,
         retention_config=retention_config,
@@ -126,6 +130,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
         verbosity=None,
         dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
         local_path='borg1',
     )
@@ -142,6 +147,24 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
         verbosity=None,
         dry_run=False,
         repository='repo',
+        storage_config={},
         retention_config=retention_config,
         remote_path='borg1',
     )
+
+
+def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    retention_config = flexmock()
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
+        BASE_PRUNE_FLAGS,
+    )
+    insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
+
+    module.prune_archives(
+        verbosity=None,
+        dry_run=False,
+        repository='repo',
+        storage_config=storage_config,
+        retention_config=retention_config,
+    )

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.15.dev'
+VERSION = '1.1.15'
 
 
 setup(