Initial work on delete/rdelete actions (#298).

NEWS

@@ -1,4 +1,5 @@
 1.8.13.dev0
+ * #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
  * #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on
    particular days of the week. See the documentation for more information:
    https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-days

borgmatic/actions/delete.py

@@ -0,0 +1,50 @@
+import logging
+
+import borgmatic.actions.arguments
+import borgmatic.borg.delete
+import borgmatic.borg.rdelete
+import borgmatic.borg.rlist
+
+logger = logging.getLogger(__name__)
+
+
+def run_delete(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "delete" action for the given repository and archive(s).
+    '''
+    if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, delete_arguments.repository
+    ):
+        logger.answer(f'{repository.get("label", repository["path"])}: Deleting archives')
+
+        archive_name = (
+            borgmatic.borg.rlist.resolve_archive_name(
+                repository['path'],
+                delete_arguments.archive,
+                config,
+                local_borg_version,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+            if delete_arguments.archive
+            else None
+        )
+
+        borgmatic.borg.delete.delete_archives(
+            repository,
+            config,
+            local_borg_version,
+            borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
+            global_arguments,
+            local_path,
+            remote_path,
+        )

borgmatic/actions/rdelete.py

@@ -0,0 +1,33 @@
+import logging
+
+import borgmatic.borg.rdelete
+
+logger = logging.getLogger(__name__)
+
+
+def run_rdelete(
+    repository,
+    config,
+    local_borg_version,
+    rdelete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Run the "rdelete" action for the given repository.
+    '''
+    if rdelete_arguments.repository is None or borgmatic.config.validate.repositories_match(
+        repository, rdelete_arguments.repository
+    ):
+        logger.answer(f'{repository.get("label", repository["path"])}: Deleting repository')
+
+        borgmatic.borg.rdelete.delete_repository(
+            repository,
+            config,
+            local_borg_version,
+            rdelete_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
+        )

borgmatic/borg/delete.py

@@ -0,0 +1,128 @@
+import argparse
+import logging
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.borg.flags
+import borgmatic.borg.rdelete
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def make_delete_command(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the delete action as an argparse.Namespace, and global arguments, return a command
+    as a tuple to delete archives from the repository.
+    '''
+    return (
+        (local_path, 'delete')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives)
+        + (
+            (('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
+            if delete_arguments.force
+            else ()
+        )
+        # Ignore match_archives and archive_name_format options from configuration, so the user has
+        # to be explicit on the command-line about the archives they want to delete.
+        + borgmatic.borg.flags.make_match_archives_flags(
+            delete_arguments.match_archives or delete_arguments.archive,
+            archive_name_format=None,
+            local_borg_version=local_borg_version,
+            default_archive_name_format='*',
+        )
+        + borgmatic.borg.flags.make_flags_from_arguments(
+            delete_arguments,
+            excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'),
+        )
+        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
+    )
+
+
+ARCHIVE_RELATED_ARGUMENT_NAMES = (
+    'archive',
+    'match_archives',
+    'first',
+    'last',
+    'oldest',
+    'newest',
+    'older',
+    'newer',
+)
+
+
+def delete_archives(
+    repository,
+    config,
+    local_borg_version,
+    delete_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the delete action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, delete the selected archives from the
+    repository. If no archives are selected, then delete the entire repository.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    if not any(
+        getattr(delete_arguments, argument_name) for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
+    ):
+        if borgmatic.borg.feature.available(
+            borgmatic.borg.feature.Feature.RDELETE, local_borg_version
+        ):
+            logger.warning(
+                'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the rdelete action instead.'
+            )
+
+        rdelete_arguments = argparse.Namespace(
+            repository=repository['path'],
+            list_archives=delete_arguments.list_archives,
+            force=delete_arguments.force,
+            cache_only=delete_arguments.cache_only,
+            keep_security_info=delete_arguments.keep_security_info,
+        )
+        return borgmatic.borg.rdelete.delete_repository(
+            repository,
+            config,
+            local_borg_version,
+            rdelete_arguments,
+            global_arguments,
+            local_path,
+            remote_path,
+        )
+
+    command = make_delete_command(
+        repository,
+        config,
+        local_borg_version,
+        delete_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.ANSWER,
+        extra_environment=borgmatic.borg.environment.make_environment(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/borg/feature.py

@@ -13,8 +13,9 @@ class Feature(Enum):
     RCREATE = 7
     RLIST = 8
     RINFO = 9
-    MATCH_ARCHIVES = 10
-    EXCLUDED_FILES_MINUS = 11
+    RDELETE = 10
+    MATCH_ARCHIVES = 11
+    EXCLUDED_FILES_MINUS = 12
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
@@ -27,6 +28,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
     Feature.RCREATE: parse('2.0.0a2'),  # borg rcreate
     Feature.RLIST: parse('2.0.0a2'),  # borg rlist
     Feature.RINFO: parse('2.0.0a2'),  # borg rinfo
+    Feature.RDELETE: parse('2.0.0a2'),  # borg rdelete
     Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
     Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
 }

borgmatic/borg/flags.py

@@ -66,7 +66,12 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
 DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'  # noqa: FS003
 
 
-def make_match_archives_flags(match_archives, archive_name_format, local_borg_version):
+def make_match_archives_flags(
+    match_archives,
+    archive_name_format,
+    local_borg_version,
+    default_archive_name_format=DEFAULT_ARCHIVE_NAME_FORMAT,
+):
     '''
     Return match archives flags based on the given match archives value, if any. If it isn't set,
     return match archives flags to match archives created with the given (or default) archive name
@@ -83,7 +88,7 @@ def make_match_archives_flags(match_archives, archive_name_format, local_borg_ve
             return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
 
     derived_match_archives = re.sub(
-        r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or DEFAULT_ARCHIVE_NAME_FORMAT
+        r'\{(now|utcnow|pid)([:%\w\.-]*)\}', '*', archive_name_format or default_archive_name_format
     )
 
     if derived_match_archives == '*':

borgmatic/borg/list.py

@@ -144,12 +144,12 @@ def list_archive(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a configuration dict, the local Borg version, global
-    arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
-    and local and remote Borg paths, display the output of listing the files of a Borg archive (or
-    return JSON output). If list_arguments.find_paths are given, list the files by searching across
-    multiple archives. If neither find_paths nor archive name are given, instead list the archives
-    in the given repository.
+    Given a local or remote repository path, a configuration dict, the local Borg version, the
+    arguments to the list action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, display the output of listing the files of
+    a Borg archive (or return JSON output). If list_arguments.find_paths are given, list the files
+    by searching across multiple archives. If neither find_paths nor archive name are given, instead
+    list the archives in the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 

borgmatic/borg/rdelete.py

@@ -0,0 +1,91 @@
+import logging
+
+import borgmatic.borg.environment
+import borgmatic.borg.feature
+import borgmatic.borg.flags
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def make_rdelete_command(
+    repository,
+    config,
+    local_borg_version,
+    rdelete_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the rdelete action as an argparse.Namespace, and global arguments, return a command
+    as a tuple to rdelete the entire repository.
+    '''
+    return (
+        (local_path,)
+        + (
+            ('rdelete',)
+            if borgmatic.borg.feature.available(
+                borgmatic.borg.feature.Feature.RDELETE, local_borg_version
+            )
+            else ('delete',)
+        )
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + borgmatic.borg.flags.make_flags('remote-path', remote_path)
+        + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json)
+        + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
+        + borgmatic.borg.flags.make_flags('list', rdelete_arguments.list_archives)
+        + (
+            (('--force',) + (('--force',) if rdelete_arguments.force >= 2 else ()))
+            if rdelete_arguments.force
+            else ()
+        )
+        + borgmatic.borg.flags.make_flags_from_arguments(
+            rdelete_arguments, excludes=('list_archives', 'force', 'repository')
+        )
+        + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
+    )
+
+
+def delete_repository(
+    repository,
+    config,
+    local_borg_version,
+    rdelete_arguments,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a local or remote repository dict, a configuration dict, the local Borg version, the
+    arguments to the rdelete action as an argparse.Namespace, global arguments as an
+    argparse.Namespace, and local and remote Borg paths, rdelete the entire repository.
+    '''
+    borgmatic.logger.add_custom_log_levels()
+
+    command = make_rdelete_command(
+        repository,
+        config,
+        local_borg_version,
+        rdelete_arguments,
+        global_arguments,
+        local_path,
+        remote_path,
+    )
+
+    borgmatic.execute.execute_command(
+        command,
+        output_log_level=logging.ANSWER,
+        # Don't capture output when Borg is expected to prompt for interactive confirmation, or the
+        # prompt won't work.
+        output_file=(
+            None
+            if rdelete_arguments.force or rdelete_arguments.cache_only
+            else borgmatic.execute.DO_NOT_CAPTURE
+        ),
+        extra_environment=borgmatic.borg.environment.make_environment(config),
+        borg_local_path=local_path,
+        borg_exit_codes=config.get('borg_exit_codes'),
+    )

borgmatic/commands/arguments.py

@@ -12,11 +12,13 @@ ACTION_ALIASES = {
     'create': ['-C'],
     'check': ['-k'],
     'config': [],
+    'delete': [],
     'extract': ['-x'],
     'export-tar': [],
     'mount': ['-m'],
     'umount': ['-u'],
     'restore': ['-r'],
+    'rdelete': [],
     'rlist': [],
     'list': ['-l'],
     'rinfo': [],
@@ -538,7 +540,7 @@ def make_parsers():
         dest='stats',
         default=False,
         action='store_true',
-        help='Display statistics of archive',
+        help='Display statistics of the pruned archive',
     )
     prune_group.add_argument(
         '--list', dest='list_archives', action='store_true', help='List archives kept/pruned'
@@ -689,6 +691,97 @@ def make_parsers():
     )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
+    delete_parser = action_parsers.add_parser(
+        'delete',
+        aliases=ACTION_ALIASES['delete'],
+        help='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
+        description='Delete an archive from a repository or delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
+        add_help=False,
+    )
+    delete_group = delete_parser.add_argument_group('delete arguments')
+    delete_group.add_argument(
+        '--repository',
+        help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one',
+    )
+    delete_group.add_argument(
+        '--archive',
+        help='Archive to delete',
+    )
+    delete_group.add_argument(
+        '--list',
+        dest='list_archives',
+        action='store_true',
+        help='Show details for the deleted archives',
+    )
+    delete_group.add_argument(
+        '--stats',
+        action='store_true',
+        help='Display statistics for the deleted archives',
+    )
+    delete_group.add_argument(
+        '--cache-only',
+        action='store_true',
+        help='Delete only the local cache for the given repository',
+    )
+    delete_group.add_argument(
+        '--force',
+        action='count',
+        help='Force deletion of corrupted archives, can be given twice if once does not work',
+    )
+    delete_group.add_argument(
+        '--keep-security-info',
+        action='store_true',
+        help='Do not delete the local security info when deleting a repository',
+    )
+    delete_group.add_argument(
+        '--save-space',
+        action='store_true',
+        help='Work slower, but using less space',
+    )
+    delete_group.add_argument(
+        '--checkpoint-interval',
+        type=int,
+        metavar='SECONDS',
+        help='Write a checkpoint at the given interval, defaults to 1800 seconds (30 minutes)',
+    )
+    delete_group.add_argument(
+        '-a',
+        '--match-archives',
+        '--glob-archives',
+        metavar='PATTERN',
+        help='Only delete archives matching this pattern',
+    )
+    delete_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    delete_group.add_argument(
+        '--first', metavar='N', help='Delete first N archives after other filters are applied'
+    )
+    delete_group.add_argument(
+        '--last', metavar='N', help='Delete last N archives after other filters are applied'
+    )
+    delete_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='Delete archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    delete_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='Delete archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    delete_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='Delete archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    delete_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='Delete archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    delete_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
     extract_parser = action_parsers.add_parser(
         'extract',
         aliases=ACTION_ALIASES['extract'],
@@ -977,6 +1070,43 @@ def make_parsers():
     )
     umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
+    rdelete_parser = action_parsers.add_parser(
+        'rdelete',
+        aliases=ACTION_ALIASES['rdelete'],
+        help='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
+        description='Delete an entire repository (with Borg 1.2+, you must run compact afterwards to actually free space)',
+        add_help=False,
+    )
+    rdelete_group = rdelete_parser.add_argument_group('delete arguments')
+    rdelete_group.add_argument(
+        '--repository',
+        help='Path of repository to delete, defaults to the configured repository if there is only one',
+    )
+    rdelete_group.add_argument(
+        '--list',
+        dest='list_archives',
+        action='store_true',
+        help='Show details for the archives in the given repository',
+    )
+    rdelete_group.add_argument(
+        '--force',
+        action='count',
+        help='Force deletion of corrupted archives, can be given twice if once does not work',
+    )
+    rdelete_group.add_argument(
+        '--cache-only',
+        action='store_true',
+        help='Delete only the local cache for the given repository',
+    )
+    rdelete_group.add_argument(
+        '--keep-security-info',
+        action='store_true',
+        help='Do not delete the local security info when deleting a repository',
+    )
+    rdelete_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
     restore_parser = action_parsers.add_parser(
         'restore',
         aliases=ACTION_ALIASES['restore'],

borgmatic/commands/borgmatic.py

@@ -18,6 +18,7 @@ import borgmatic.actions.config.bootstrap
 import borgmatic.actions.config.generate
 import borgmatic.actions.config.validate
 import borgmatic.actions.create
+import borgmatic.actions.delete
 import borgmatic.actions.export_key
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
@@ -26,6 +27,7 @@ import borgmatic.actions.list
 import borgmatic.actions.mount
 import borgmatic.actions.prune
 import borgmatic.actions.rcreate
+import borgmatic.actions.rdelete
 import borgmatic.actions.restore
 import borgmatic.actions.rinfo
 import borgmatic.actions.rlist
@@ -479,6 +481,26 @@ def run_actions(
                 local_path,
                 remote_path,
             )
+        elif action_name == 'delete' and action_name not in skip_actions:
+            borgmatic.actions.delete.run_delete(
+                repository,
+                config,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
+        elif action_name == 'rdelete' and action_name not in skip_actions:
+            borgmatic.actions.rdelete.run_rdelete(
+                repository,
+                config,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
+            )
         elif action_name == 'borg' and action_name not in skip_actions:
             borgmatic.actions.borg.run_borg(
                 repository,

borgmatic/config/schema.yaml

@@ -747,11 +747,13 @@ properties:
                 - compact
                 - create
                 - check
+                - delete
                 - extract
                 - config
                 - export-tar
                 - mount
                 - umount
+                - rdelete
                 - restore
                 - rlist
                 - list

docs/Dockerfile

@@ -4,7 +4,7 @@ COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" "config validate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
+    && for action in rcreate transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount rdelete restore rlist list rinfo info break-lock borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic $action --help >> /command-line.txt; done
 

tests/unit/borg/test_flags.py

@@ -190,6 +190,20 @@ def test_make_match_archives_flags_makes_flags_with_globs(
     )
 
 
+def test_make_match_archives_flags_accepts_default_archive_name_format():
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    assert (
+        module.make_match_archives_flags(
+            match_archives=None,
+            archive_name_format=None,
+            local_borg_version=flexmock(),
+            default_archive_name_format='*',
+        )
+        == ()
+    )
+
+
 def test_warn_for_aggressive_archive_flags_without_archive_flags_bails():
     flexmock(module.logger).should_receive('warning').never()