Merge branch 'feature/list-formatting' of https://github.com/manwegit/borg into feature/list-formatting

borg/archiver.py

@@ -15,8 +15,8 @@ import textwrap
 import traceback
 
 from . import __version__
-from .helpers import Error, location_validator, archivename_validator, format_time, format_file_size, \
-    parse_pattern, PathPrefixPattern, to_localtime, timestamp, \
+from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \
+    parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
     get_cache_dir, get_keys_dir, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
@@ -442,6 +442,19 @@ class Archiver:
                 for item in archive.iter_items():
                     print(remove_surrogates(item[b'path']))
             else:
+                long_format = "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}"
+                user_format = long_format
+                """use_user_format flag is used to speed up default listing.
+                When user issues format options, listing is a bit slower, but more keys are available and
+                precalculated
+                """
+                use_user_format = False
+                if args.listformat:
+                    user_format = args.listformat
+                    use_user_format = True
+
+                archive_name = archive.name
+
                 for item in archive.iter_items():
                     mode = stat.filemode(item[b'mode'])
                     type = mode[0]
@@ -451,12 +464,14 @@ class Archiver:
                             size = sum(size for _, size, _ in item[b'chunks'])
                         except KeyError:
                             pass
-                    try:
-                        mtime = datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9)
-                    except OverflowError:
-                        # likely a broken mtime and datetime did not want to go beyond year 9999
-                        mtime = datetime(9999, 12, 31, 23, 59, 59)
+                    mtime = safe_timestamp(item[b'mtime'])
+
+                    if use_user_format:
+                        atime = safe_timestamp(item[b'atime'])
+                        ctime = safe_timestamp(item[b'ctime'])
+
                     if b'source' in item:
+                        source = item[b'source']
                         if type == 'l':
                             extra = ' -> %s' % item[b'source']
                         else:
@@ -464,10 +479,46 @@ class Archiver:
                             extra = ' link to %s' % item[b'source']
                     else:
                         extra = ''
-                    print('%s %-6s %-6s %8d %s %s%s' % (
-                        mode, item[b'user'] or item[b'uid'],
-                        item[b'group'] or item[b'gid'], size, format_time(mtime),
-                        remove_surrogates(item[b'path']), extra))
+                        source = ''
+
+                    item_data = {
+                            'mode': mode,
+                            'user': item[b'user'] or item[b'uid'],
+                            'group': item[b'group'] or item[b'gid'],
+                            'size': size,
+                            'isomtime': format_time(mtime),
+                            'path': remove_surrogates(item[b'path']),
+                            'extra': extra,
+                            }
+                    if use_user_format:
+                        item_data_advanced = {
+                            'bmode': item[b'mode'],
+                            'type': type,
+                            'source': source,
+                            'linktarget': source,
+                            'uid': item[b'uid'],
+                            'gid': item[b'gid'],
+                            'mtime': mtime,
+                            'isoctime': format_time(ctime),
+                            'ctime': ctime,
+                            'isoatime': format_time(atime),
+                            'atime': atime,
+                            'archivename': archive_name,
+                            'SPACE': " ",
+                            'TAB': "\t",
+                            'LF': "\n",
+                            'CR': "\r",
+                            'NEWLINE': os.linesep,
+                            'formatkeys': ()
+                            }
+                        item_data_advanced["formatkeys"] = list(item_data.keys())
+                        item_data.update(item_data_advanced)
+
+                    if use_user_format:
+                        print(format_line(user_format, item_data), end='')
+                    else:
+                        print(format_line(user_format, item_data))
+
         else:
             for archive_info in manifest.list_archive_infos(sort_by='ts'):
                 if args.prefix and not archive_info.name.startswith(args.prefix):
@@ -1096,6 +1147,10 @@ class Archiver:
         subparser.add_argument('--short', dest='short',
                                action='store_true', default=False,
                                help='only print file/directory names, nothing else')
+        subparser.add_argument('--list-format', dest='listformat', type=str,
+                               help="""specify format for archive file listing
+                                (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
+                                Special "{formatkeys}" exists to list available keys""")
         subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
                                help='only consider archive names starting with this prefix')
         subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',

borg/helpers.py

@@ -519,6 +519,29 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
     return tag_paths
 
 
+def format_line(format, data):
+    # TODO: Filter out unwanted properties of str.format(), because "format" is user provided.
+
+    try:
+        return format.format(**data)
+    except (KeyError, ValueError) as e:
+        # this should catch format errors
+        print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
+    except:
+        # something unexpected, print error and raise exception
+        print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
+        raise
+    return ''
+
+
+def safe_timestamp(item_timestamp_ns):
+    try:
+        return datetime.fromtimestamp(bigint_to_int(item_timestamp_ns) / 1e9)
+    except OverflowError:
+        # likely a broken file time and datetime did not want to go beyond year 9999
+        return datetime(9999, 12, 31, 23, 59, 59)
+
+
 def format_time(t):
     """use ISO-8601 date and time format
     """

borg/testsuite/archiver.py

@@ -892,6 +892,16 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.assert_in('test-2', output)
         self.assert_not_in('something-else', output)
 
+    def test_list_list_format(self):
+        self.cmd('init', self.repository_location)
+        test_archive = self.repository_location + '::test'
+        self.cmd('create', test_archive, src_dir)
+        output_1 = self.cmd('list', test_archive)
+        output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
+        output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive)
+        self.assertEqual(output_1, output_2)
+        self.assertNotEqual(output_1, output_3)
+
     def test_break_lock(self):
         self.cmd('init', self.repository_location)
         self.cmd('break-lock', self.repository_location)

docs/usage.rst

@@ -336,6 +336,23 @@ Examples
     -rwxr-xr-x root   root       2140 Fri, 2015-03-27 20:24:22 bin/bzdiff
     ...
 
+    $ borg list /mnt/backup::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
+    drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 .
+    drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 code
+    drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 code/myproject
+    -rw-rw-r-- user   user    1416192 Sun, 2015-02-01 11:00:00 code/myproject/file.ext
+    ...
+
+    # see what is changed between archives, based on file modification time, size and file path
+    $ borg list /mnt/backup::archiveA --list-format="{mtime:%s}{TAB}{size}{TAB}{path}{LF}" |sort -n > /tmp/list.archiveA
+    $ borg list /mnt/backup::archiveB --list-format="{mtime:%s}{TAB}{size}{TAB}{path}{LF}" |sort -n > /tmp/list.archiveB
+    $ diff -y /tmp/list.archiveA /tmp/list.archiveB
+    1422781200      0       .                                       1422781200      0       .
+    1422781200      0       code                                    1422781200      0       code
+    1422781200      0       code/myproject                          1422781200      0       code/myproject
+    1422781200      1416192 code/myproject/file.ext               | 1454664653      1416192 code/myproject/file.ext
+    ...
+
 
 .. include:: usage/delete.rst.inc