Compact repository segments with new "borgmatic compact" action (#394).

NEWS

@@ -1,4 +1,5 @@
 1.5.23.dev0
+ * #394: Compact repository segments with new "borgmatic compact" action. Borg 1.2+ only.
  * #480, #482: Fix traceback when a YAML validation error occurs.
 
 1.5.22

borgmatic/borg/compact.py

@@ -0,0 +1,42 @@
+import logging
+
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def compact_segments(
+    dry_run,
+    repository,
+    storage_config,
+    retention_config,
+    local_path='borg',
+    remote_path=None,
+    progress=False,
+    cleanup_commits=False,
+    threshold=None,
+):
+    '''
+    Given dry-run flag, a local or remote repository path, a storage config dict, and a
+    retention config dict, compact Borg segments in a repository.
+    '''
+    umask = storage_config.get('umask', None)
+    lock_wait = storage_config.get('lock_wait', None)
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '')
+
+    full_command = (
+        (local_path, 'compact')
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--umask', str(umask)) if umask else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+        + (('--progress',) if progress else ())
+        + (('--cleanup-commits',) if cleanup_commits else ())
+        + (('--threshold', str(threshold)) if threshold else ())
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--dry-run',) if dry_run else ())
+        + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+        + (repository,)
+    )
+
+    execute_command(full_command, output_log_level=logging.WARNING, borg_local_path=local_path)

borgmatic/commands/arguments.py

@@ -6,6 +6,7 @@ from borgmatic.config import collect
 SUBPARSER_ALIASES = {
     'init': ['--init', '-I'],
     'prune': ['--prune', '-p'],
+    'compact': [],
     'create': ['--create', '-C'],
     'check': ['--check', '-k'],
     'extract': ['--extract', '-x'],
@@ -258,6 +259,36 @@ def parse_arguments(*unparsed_arguments):
     )
     prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
+    compact_parser = subparsers.add_parser(
+        'compact',
+        aliases=SUBPARSER_ALIASES['compact'],
+        help='compact segments to free space (Borg 1.2+ only)',
+        description='compact segments to free space (Borg 1.2+ only)',
+        add_help=False,
+    )
+    compact_group = compact_parser.add_argument_group('compact arguments')
+    compact_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress as each segment is compacted',
+    )
+    compact_group.add_argument(
+        '--cleanup-commits',
+        dest='cleanup_commits',
+        default=False,
+        action='store_true',
+        help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1',
+    )
+    compact_group.add_argument(
+        '--threshold',
+        type=int,
+        dest='threshold',
+        help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
+    )
+    compact_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
     create_parser = subparsers.add_parser(
         'create',
         aliases=SUBPARSER_ALIASES['create'],

borgmatic/commands/borgmatic.py

@@ -13,6 +13,7 @@ import pkg_resources
 
 from borgmatic.borg import borg as borg_borg
 from borgmatic.borg import check as borg_check
+from borgmatic.borg import compact as borg_compact
 from borgmatic.borg import create as borg_create
 from borgmatic.borg import environment as borg_environment
 from borgmatic.borg import export_tar as borg_export_tar
@@ -80,6 +81,14 @@ def run_configuration(config_filename, config, arguments):
                 'pre-prune',
                 global_arguments.dry_run,
             )
+        if 'compact' in arguments:
+            command.execute_hook(
+                hooks.get('before_compact'),
+                hooks.get('umask'),
+                config_filename,
+                'pre-compact',
+                global_arguments.dry_run,
+            )
         if 'create' in arguments:
             command.execute_hook(
                 hooks.get('before_backup'),
@@ -169,6 +178,14 @@ def run_configuration(config_filename, config, arguments):
                     'post-prune',
                     global_arguments.dry_run,
                 )
+            if 'compact' in arguments:
+                command.execute_hook(
+                    hooks.get('after_compact'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'post-compact',
+                    global_arguments.dry_run,
+                )
             if 'create' in arguments:
                 dispatch.call_hooks(
                     'remove_database_dumps',
@@ -314,6 +331,19 @@ def run_actions(
             stats=arguments['prune'].stats,
             files=arguments['prune'].files,
         )
+    if 'compact' in arguments:
+        logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
+        borg_compact.compact_segments(
+            global_arguments.dry_run,
+            repository,
+            storage,
+            retention,
+            local_path=local_path,
+            remote_path=remote_path,
+            progress=arguments['compact'].progress,
+            cleanup_commits=arguments['compact'].cleanup_commits,
+            threshold=arguments['compact'].threshold,
+        )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
         dispatch.call_hooks(

borgmatic/config/schema.yaml

@@ -353,6 +353,11 @@ properties:
                         description: |
                             Extra command-line options to pass to "borg prune".
                         example: "--save-space"
+                    compact:
+                        type: string
+                        description: |
+                            Extra command-line options to pass to "borg compact".
+                        example: "--save-space"
                     create:
                         type: string
                         description: |
@@ -522,6 +527,15 @@ properties:
                     before pruning, run once per configuration file.
                 example:
                     - echo "Starting pruning."
+            before_compact:
+                type: array
+                items:
+                    type: string
+                description: |
+                    List of one or more shell commands or scripts to execute
+                    before compaction, run once per configuration file.
+                example:
+                    - echo "Starting compaction."
             before_check:
                 type: array
                 items:
@@ -549,6 +563,15 @@ properties:
                     after creating a backup, run once per configuration file.
                 example:
                     - echo "Finished a backup."
+            after_compact:
+                type: array
+                items:
+                    type: string
+                description: |
+                    List of one or more shell commands or scripts to execute
+                    after compaction, run once per configuration file.
+                example:
+                    - echo "Finished compaction."
             after_prune:
                 type: array
                 items: