Merge pull request #1179 from Abogical/list-format

Apply --format when listing repository archives, fixes #1166

docs/usage/list.rst.inc

@@ -35,8 +35,22 @@ This command lists the contents of a repository or an archive.
 
 See the "borg help patterns" command for more help on exclude patterns.
 
-The following keys are available for --format when listing files:
+The following keys are available for --format:
+ - NEWLINE: OS dependent line separator
+ - NL: alias of NEWLINE
+ - NUL: NUL character for creating print0 / xargs -0 like output, see barchive/bpath
+ - SPACE
+ - TAB
+ - CR
+ - LF
+
+-- Keys for listing repository archives:
+ - archive: archive name interpreted as text (might be missing non-text characters, see barchive)
+ - barchive: verbatim archive name, can contain any character except NUL
+ - time: time of creation of the archive
+ - id: internal ID of the archive
 
+-- Keys for listing archive files:
  - type
  - mode
  - uid
@@ -47,6 +61,7 @@ The following keys are available for --format when listing files:
  - bpath: verbatim POSIX path, can contain any character except NUL
  - source: link target for links (identical to linktarget)
  - linktarget
+ - flags
 
  - size
  - csize: compressed size
@@ -70,11 +85,3 @@ The following keys are available for --format when listing files:
  - archiveid
  - archivename
  - extra: prepends {source} with " -> " for soft links and " link to " for hard links
-
- - NEWLINE: OS dependent line separator
- - NL: alias of NEWLINE
- - NUL: NUL character for creating print0 / xargs -0 like ouput, see bpath
- - SPACE
- - TAB
- - CR
- - LF

src/borg/archiver.py

@@ -29,7 +29,7 @@ from .constants import *  # NOQA
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .helpers import Error, NoManifestError
 from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec
-from .helpers import ItemFormatter, format_time, format_file_size, format_archive
+from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive
 from .helpers import safe_encode, remove_surrogates, bin_to_hex
 from .helpers import prune_within, prune_split
 from .helpers import to_localtime, timestamp
@@ -738,6 +738,14 @@ class Archiver:
     @with_repository()
     def do_list(self, args, repository, manifest, key):
         """List archive or repository contents"""
+        if not hasattr(sys.stdout, 'buffer'):
+            # This is a shim for supporting unit tests replacing sys.stdout with e.g. StringIO,
+            # which doesn't have an underlying buffer (= lower file object).
+            def write(bytestring):
+                sys.stdout.write(bytestring.decode('utf-8', errors='replace'))
+        else:
+            write = sys.stdout.buffer.write
+
         if args.location.archive:
             matcher, _ = self.build_matcher(args.excludes, args.paths)
             with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
@@ -751,23 +759,22 @@ class Archiver:
                     format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}"
                 formatter = ItemFormatter(archive, format)
 
-                if not hasattr(sys.stdout, 'buffer'):
-                    # This is a shim for supporting unit tests replacing sys.stdout with e.g. StringIO,
-                    # which doesn't have an underlying buffer (= lower file object).
-                    def write(bytestring):
-                        sys.stdout.write(bytestring.decode('utf-8', errors='replace'))
-                else:
-                    write = sys.stdout.buffer.write
                 for item in archive.iter_items(lambda item: matcher.match(item.path)):
                     write(safe_encode(formatter.format_item(item)))
         else:
+            if args.format:
+                format = args.format
+            elif args.short:
+                format = "{archive}{NL}"
+            else:
+                format = "{archive:<36} {time} [{id}]{NL}"
+            formatter = ArchiveFormatter(format)
+
             for archive_info in manifest.list_archive_infos(sort_by='ts'):
                 if args.prefix and not archive_info.name.startswith(args.prefix):
                     continue
-                if args.short:
-                    print(archive_info.name)
-                else:
-                    print(format_archive(archive_info))
+                write(safe_encode(formatter.format_item(archive_info)))
+
         return self.exit_code
 
     @with_repository(cache=True)
@@ -1618,8 +1625,13 @@ class Archiver:
 
         See the "borg help patterns" command for more help on exclude patterns.
 
-        The following keys are available for --format when listing files:
+        The following keys are available for --format:
+        """) + BaseFormatter.keys_help() + textwrap.dedent("""
+
+        -- Keys for listing repository archives:
+        """) + ArchiveFormatter.keys_help() + textwrap.dedent("""
 
+        -- Keys for listing archive files:
         """) + ItemFormatter.keys_help()
         subparser = subparsers.add_parser('list', parents=[common_parser], add_help=False,
                                           description=self.do_list.__doc__,

src/borg/helpers.py

@@ -1132,7 +1132,7 @@ def log_multi(*msgs, level=logging.INFO, logger=logger):
         logger.log(level, line)
 
 
-class ItemFormatter:
+class BaseFormatter:
     FIXED_KEYS = {
         # Formatting aids
         'LF': '\n',
@@ -1143,19 +1143,54 @@ class ItemFormatter:
         'NEWLINE': os.linesep,
         'NL': os.linesep,
     }
+
+    def get_item_data(self, item):
+        raise NotImplementedError
+
+    def format_item(self, item):
+        return self.format.format_map(self.get_item_data(item))
+
+    @staticmethod
+    def keys_help():
+        return " - NEWLINE: OS dependent line separator\n" \
+               " - NL: alias of NEWLINE\n" \
+               " - NUL: NUL character for creating print0 / xargs -0 like output, see barchive/bpath\n" \
+               " - SPACE\n" \
+               " - TAB\n" \
+               " - CR\n" \
+               " - LF"
+
+
+class ArchiveFormatter(BaseFormatter):
+
+    def __init__(self, format):
+        self.format = partial_format(format, self.FIXED_KEYS)
+
+    def get_item_data(self, archive):
+        return {
+            'barchive': archive.name,
+            'archive': remove_surrogates(archive.name),
+            'id': bin_to_hex(archive.id),
+            'time': format_time(to_localtime(archive.ts)),
+        }
+
+    @staticmethod
+    def keys_help():
+        return " - archive: archive name interpreted as text (might be missing non-text characters, see barchive)\n" \
+               " - barchive: verbatim archive name, can contain any character except NUL\n" \
+               " - time: time of creation of the archive\n" \
+               " - id: internal ID of the archive"
+
+
+class ItemFormatter(BaseFormatter):
     KEY_DESCRIPTIONS = {
         'bpath': 'verbatim POSIX path, can contain any character except NUL',
         'path': 'path interpreted as text (might be missing non-text characters, see bpath)',
         'source': 'link target for links (identical to linktarget)',
         'extra': 'prepends {source} with " -> " for soft links and " link to " for hard links',
-
         'csize': 'compressed size',
         'num_chunks': 'number of chunks in this file',
         'unique_chunks': 'number of unique chunks in this file',
-
-        'NEWLINE': 'OS dependent line separator',
-        'NL': 'alias of NEWLINE',
-        'NUL': 'NUL character for creating print0 / xargs -0 like ouput, see bpath',
     }
     KEY_GROUPS = (
         ('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'flags'),
@@ -1163,7 +1198,6 @@ class ItemFormatter:
         ('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'),
         tuple(sorted(hashlib.algorithms_guaranteed)),
         ('archiveid', 'archivename', 'extra'),
-        ('NEWLINE', 'NL', 'NUL', 'SPACE', 'TAB', 'CR', 'LF'),
     )
 
     @classmethod
@@ -1183,6 +1217,9 @@ class ItemFormatter:
     def keys_help(cls):
         help = []
         keys = cls.available_keys()
+        for key in cls.FIXED_KEYS:
+            keys.remove(key)
+
         for group in cls.KEY_GROUPS:
             for key in group:
                 keys.remove(key)
@@ -1254,9 +1291,6 @@ class ItemFormatter:
             item_data[key] = self.call_keys[key](item)
         return item_data
 
-    def format_item(self, item):
-        return self.format.format_map(self.get_item_data(item))
-
     def calculate_num_chunks(self, item):
         return len(item.get('chunks', []))
 

src/borg/testsuite/archiver.py

@@ -1144,6 +1144,18 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.assertEqual(output_1, output_2)
         self.assertNotEqual(output_1, output_3)
 
+    def test_list_repository_format(self):
+        self.cmd('init', self.repository_location)
+        self.cmd('create', self.repository_location + '::test-1', src_dir)
+        self.cmd('create', self.repository_location + '::test-2', src_dir)
+        output_1 = self.cmd('list', self.repository_location)
+        output_2 = self.cmd('list', '--format', '{archive:<36} {time} [{id}]{NL}', self.repository_location)
+        self.assertEqual(output_1, output_2)
+        output_1 = self.cmd('list', '--short', self.repository_location)
+        self.assertEqual(output_1, 'test-1\ntest-2\n')
+        output_1 = self.cmd('list', '--format', '{barchive}/', self.repository_location)
+        self.assertEqual(output_1, 'test-1/test-2/')
+
     def test_list_hash(self):
         self.create_regular_file('empty_file', size=0)
         self.create_regular_file('amb', contents=b'a' * 1000000)