Merge pull request #2658 from enkore/issue/2448

implement --glob-archives/-a

src/borg/archive.py

@@ -1124,19 +1124,19 @@ class ArchiveChecker:
         self.error_found = False
         self.possibly_superseded = set()
 
-    def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', prefix='',
+    def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by='', glob=None,
               verify_data=False, save_space=False):
         """Perform a set of checks on 'repository'
 
         :param repair: enable repair mode, write updated or corrected data into repository
         :param archive: only check this archive
         :param first/last/sort_by: only check this number of first/last archives ordered by sort_by
-        :param prefix: only check archives with this prefix
+        :param glob: only check archives matching this glob
         :param verify_data: integrity verification of data referenced by archives
         :param save_space: Repository.commit(save_space)
         """
         logger.info('Starting archive consistency check...')
-        self.check_all = archive is None and not any((first, last, prefix))
+        self.check_all = archive is None and not any((first, last, glob))
         self.repair = repair
         self.repository = repository
         self.init_chunks()
@@ -1158,7 +1158,7 @@ class ArchiveChecker:
                 self.error_found = True
                 del self.chunks[Manifest.MANIFEST_ID]
                 self.manifest = self.rebuild_manifest()
-        self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, prefix=prefix)
+        self.rebuild_refcounts(archive=archive, first=first, last=last, sort_by=sort_by, glob=glob)
         self.orphan_chunks_check()
         self.finish(save_space=save_space)
         if self.error_found:
@@ -1331,7 +1331,7 @@ class ArchiveChecker:
         logger.info('Manifest rebuild complete.')
         return manifest
 
-    def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', prefix=''):
+    def rebuild_refcounts(self, archive=None, first=0, last=0, sort_by='', glob=None):
         """Rebuild object reference counts by walking the metadata
 
         Missing and/or incorrect data is repaired when detected
@@ -1495,10 +1495,10 @@ class ArchiveChecker:
 
         if archive is None:
             sort_by = sort_by.split(',')
-            if any((first, last, prefix)):
-                archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last)
-                if prefix and not archive_infos:
-                    logger.warning('--prefix %s does not match any archives', prefix)
+            if any((first, last, glob)):
+                archive_infos = self.manifest.archives.list(sort_by=sort_by, glob=glob, first=first, last=last)
+                if glob and not archive_infos:
+                    logger.warning('--glob-archives %s does not match any archives', glob)
                 if first and len(archive_infos) < first:
                     logger.warning('--first %d archives: only found %d archives', first, len(archive_infos))
                 if last and len(archive_infos) < last:

src/borg/archiver.py

@@ -33,6 +33,7 @@ import msgpack
 import borg
 from . import __version__
 from . import helpers
+from . import shellpattern
 from .algorithms.checksums import crc32
 from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special
 from .archive import BackupOSError, backup_io
@@ -283,9 +284,11 @@ class Archiver:
         if not args.archives_only:
             if not repository.check(repair=args.repair, save_space=args.save_space):
                 return EXIT_WARNING
+        if args.prefix:
+            args.glob_archives = args.prefix + '*'
         if not args.repo_only and not ArchiveChecker().check(
                 repository, repair=args.repair, archive=args.location.archive,
-                first=args.first, last=args.last, sort_by=args.sort_by or 'ts', prefix=args.prefix,
+                first=args.first, last=args.last, sort_by=args.sort_by or 'ts', glob=args.glob_archives,
                 verify_data=args.verify_data, save_space=args.save_space):
             return EXIT_WARNING
         return EXIT_SUCCESS
@@ -1168,7 +1171,7 @@ class Archiver:
     @with_repository(exclusive=True, manifest=False)
     def do_delete(self, args, repository):
         """Delete an existing repository or archives"""
-        if any((args.location.archive, args.first, args.last, args.prefix)):
+        if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)):
             return self._delete_archives(args, repository)
         else:
             return self._delete_repository(args, repository)
@@ -1365,7 +1368,7 @@ class Archiver:
     @with_repository(cache=True, compatibility=(Manifest.Operation.READ,))
     def do_info(self, args, repository, manifest, key, cache):
         """Show archive details such as disk space used"""
-        if any((args.location.archive, args.first, args.last, args.prefix)):
+        if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)):
             return self._info_archives(args, repository, manifest, key, cache)
         else:
             return self._info_repository(args, repository, manifest, key, cache)
@@ -1463,7 +1466,10 @@ class Archiver:
             return self.exit_code
         archives_checkpoints = manifest.archives.list(sort_by=['ts'], reverse=True)  # just a ArchiveInfo list
         if args.prefix:
-            archives_checkpoints = [arch for arch in archives_checkpoints if arch.name.startswith(args.prefix)]
+            args.glob_archives = args.prefix + '*'
+        if args.glob_archives:
+            regex = re.compile(shellpattern.translate(args.glob_archives))
+            archives_checkpoints = [arch for arch in archives_checkpoints if regex.match(arch.name) is not None]
         is_checkpoint = re.compile(r'\.checkpoint(\.\d+)?$').search
         checkpoints = [arch for arch in archives_checkpoints if is_checkpoint(arch.name)]
         # keep the latest checkpoint, if there is no later non-checkpoint archive
@@ -3344,8 +3350,7 @@ class Archiver:
                                help='number of monthly archives to keep')
         subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
                                help='number of yearly archives to keep')
-        subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
-                               help='only consider archive names starting with this prefix')
+        self.add_archives_filters_args(subparser, sort_by=False, first_last=False)
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
                                help='work slower, but using less space')
@@ -3839,21 +3844,28 @@ class Archiver:
         return parser
 
     @staticmethod
-    def add_archives_filters_args(subparser):
+    def add_archives_filters_args(subparser, sort_by=True, first_last=True):
         filters_group = subparser.add_argument_group('filters', 'Archive filters can be applied to repository targets.')
-        filters_group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='',
-                                   help='only consider archive names starting with this prefix')
-
-        sort_by_default = 'timestamp'
-        filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default,
-                                   help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}'
-                                   .format(', '.join(HUMAN_SORT_KEYS), sort_by_default))
-
         group = filters_group.add_mutually_exclusive_group()
-        group.add_argument('--first', dest='first', metavar='N', default=0, type=int,
-                           help='consider first N archives after other filters were applied')
-        group.add_argument('--last', dest='last', metavar='N', default=0, type=int,
-                           help='consider last N archives after other filters were applied')
+        group.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec, default='',
+                           help='only consider archive names starting with this prefix.')
+        group.add_argument('-a', '--glob-archives', dest='glob_archives', default=None,
+                           help='only consider archive names matching the glob. '
+                                'sh: rules apply, see "borg help patterns". '
+                                '--prefix and --glob-archives are mutually exclusive.')
+
+        if sort_by:
+            sort_by_default = 'timestamp'
+            filters_group.add_argument('--sort-by', dest='sort_by', type=SortBySpec, default=sort_by_default,
+                                       help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}'
+                                       .format(', '.join(HUMAN_SORT_KEYS), sort_by_default))
+
+        if first_last:
+            group = filters_group.add_mutually_exclusive_group()
+            group.add_argument('--first', dest='first', metavar='N', default=0, type=int,
+                               help='consider first N archives after other filters were applied')
+            group.add_argument('--last', dest='last', metavar='N', default=0, type=int,
+                               help='consider last N archives after other filters were applied')
 
     def get_args(self, argv, cmd):
         """usually, just returns argv, except if we deal with a ssh forced command for borg serve."""

src/borg/helpers.py

@@ -42,6 +42,7 @@ from . import __version__ as borg_version
 from . import __version_tuple__ as borg_version_tuple
 from . import chunker
 from . import hashindex
+from . import shellpattern
 from .constants import *  # NOQA
 
 
@@ -189,7 +190,7 @@ class Archives(abc.MutableMapping):
         name = safe_encode(name)
         del self._archives[name]
 
-    def list(self, sort_by=(), reverse=False, prefix='', first=None, last=None):
+    def list(self, sort_by=(), reverse=False, glob=None, first=None, last=None):
         """
         Inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
         Returns list of borg.helpers.ArchiveInfo instances.
@@ -197,7 +198,8 @@ class Archives(abc.MutableMapping):
         """
         if isinstance(sort_by, (str, bytes)):
             raise TypeError('sort_by must be a sequence of str')
-        archives = [x for x in self.values() if x.name.startswith(prefix)]
+        regex = re.compile(shellpattern.translate(glob or '*'))
+        archives = [x for x in self.values() if regex.match(x.name) is not None]
         for sortkey in reversed(sort_by):
             archives.sort(key=attrgetter(sortkey))
         if reverse or last:
@@ -207,11 +209,13 @@ class Archives(abc.MutableMapping):
 
     def list_considering(self, args):
         """
-        get a list of archives, considering --first/last/prefix/sort cmdline args
+        get a list of archives, considering --first/last/prefix/glob-archives/sort cmdline args
         """
         if args.location.archive:
-            raise Error('The options --first, --last and --prefix can only be used on repository targets.')
-        return self.list(sort_by=args.sort_by.split(','), prefix=args.prefix, first=args.first, last=args.last)
+            raise Error('The options --first, --last, --prefix and --glob-archives can only be used on repository targets.')
+        if args.prefix:
+            args.glob_archives = args.prefix + '*'
+        return self.list(sort_by=args.sort_by.split(','), glob=args.glob_archives, first=args.first, last=args.last)
 
     def set_raw_dict(self, d):
         """set the dict we get from the msgpack unpacker"""

src/borg/testsuite/archiver.py

@@ -1708,6 +1708,27 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.assert_in('bar-2015-08-12-10:00', output)
         self.assert_in('bar-2015-08-12-20:00', output)
 
+    def test_prune_repository_glob(self):
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', self.repository_location + '::2015-08-12-10:00-foo', src_dir)
+        self.cmd('create', self.repository_location + '::2015-08-12-20:00-foo', src_dir)
+        self.cmd('create', self.repository_location + '::2015-08-12-10:00-bar', src_dir)
+        self.cmd('create', self.repository_location + '::2015-08-12-20:00-bar', src_dir)
+        output = self.cmd('prune', '--list', '--dry-run', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo')
+        self.assert_in('Keeping archive: 2015-08-12-20:00-foo', output)
+        self.assert_in('Would prune:     2015-08-12-10:00-foo', output)
+        output = self.cmd('list', self.repository_location)
+        self.assert_in('2015-08-12-10:00-foo', output)
+        self.assert_in('2015-08-12-20:00-foo', output)
+        self.assert_in('2015-08-12-10:00-bar', output)
+        self.assert_in('2015-08-12-20:00-bar', output)
+        self.cmd('prune', self.repository_location, '--keep-daily=2', '--glob-archives=2015-*-foo')
+        output = self.cmd('list', self.repository_location)
+        self.assert_not_in('2015-08-12-10:00-foo', output)
+        self.assert_in('2015-08-12-20:00-foo', output)
+        self.assert_in('2015-08-12-10:00-bar', output)
+        self.assert_in('2015-08-12-20:00-bar', output)
+
     def test_list_prefix(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.cmd('create', self.repository_location + '::test-1', src_dir)