Merge pull request #2965 from ThomasWaldmann/issue/2376-1.1

Issue/2376 PR backported to 1.1

docs/changes.rst

@@ -131,6 +131,19 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
+Version 1.1.0rc2 (not released yet)
+------------------------------------
+
+Compatibility notes:
+
+- list: corrected mix-up of "isomtime" and "mtime" formats. Previously,
+  "isomtime" was the default but produced a verbose human format,
+  while "mtime" produced a ISO-8601-like format.
+  The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like),
+  and the default is now "mtime".
+  "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space).
+
+
 Version 1.1.0rc1 (2017-07-24)
 -----------------------------
 

docs/internals/frontends.rst

@@ -209,8 +209,9 @@ Standard output
 *stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create`
 and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object.
 
-Dates are formatted according to ISO-8601 with the strftime format string '%a, %Y-%m-%d %H:%M:%S',
-e.g. *Sat, 2016-02-25 23:50:06*.
+Dates are formatted according to ISO 8601 in local time. No explicit time zone is specified *at this time*
+(subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S.%f',
+e.g. ``2017-08-07T12:27:20.123456``.
 
 The root object at least contains a *repository* key with an object containing:
 
@@ -267,7 +268,7 @@ Example *borg info* output::
         },
         "repository": {
             "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
-            "last_modified": "Mon, 2017-02-27 21:21:58",
+            "last_modified": "2017-08-07T12:27:20.789123",
             "location": "/home/user/testrepo"
         },
         "security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
@@ -328,7 +329,7 @@ Example of a simple archive listing (``borg list --last 1 --json``)::
             {
                 "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a",
                 "name": "host-system-backup-2017-02-27",
-                "start": "Mon, 2017-02-27 21:21:52"
+                "start": "2017-08-07T12:27:20.789123"
             }
         ],
         "encryption": {
@@ -336,7 +337,7 @@ Example of a simple archive listing (``borg list --last 1 --json``)::
         },
         "repository": {
             "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
-            "last_modified": "Mon, 2017-02-27 21:21:58",
+            "last_modified": "2017-08-07T12:27:20.789123",
             "location": "/home/user/repository"
         }
     }
@@ -354,14 +355,14 @@ The same archive with more information (``borg info --last 1 --json``)::
                 ],
                 "comment": "",
                 "duration": 5.641542,
-                "end": "Mon, 2017-02-27 21:21:58",
+                "end": "2017-02-27T12:27:20.789123",
                 "hostname": "host",
                 "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a",
                 "limits": {
                     "max_archive_size": 0.0001330855110409714
                 },
                 "name": "host-system-backup-2017-02-27",
-                "start": "Mon, 2017-02-27 21:21:52",
+                "start": "2017-02-27T12:27:20.789123",
                 "stats": {
                     "compressed_size": 1880961894,
                     "deduplicated_size": 2791,
@@ -387,7 +388,7 @@ The same archive with more information (``borg info --last 1 --json``)::
         },
         "repository": {
             "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23",
-            "last_modified": "Mon, 2017-02-27 21:21:58",
+            "last_modified": "2017-08-07T12:27:20.789123",
             "location": "/home/user/repository"
         }
     }
@@ -405,8 +406,8 @@ Refer to the *borg list* documentation for the available keys and their meaning.
 
 Example (excerpt) of ``borg list --json-lines``::
 
-    {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0}
-    {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0}
+    {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0}
+    {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0}
 
 .. _msgid:
 

src/borg/archive.py

@@ -32,7 +32,7 @@ from .helpers import ChunkIteratorFileWrapper, open_item
 from .helpers import Error, IntegrityError, set_ec
 from .helpers import uid2user, user2uid, gid2group, group2gid
 from .helpers import parse_timestamp, to_localtime
-from .helpers import format_time, format_timedelta, format_file_size, file_status, FileSize
+from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
 from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
 from .helpers import StableDict
 from .helpers import bin_to_hex
@@ -381,8 +381,8 @@ class Archive:
         info = {
             'name': self.name,
             'id': self.fpr,
-            'start': format_time(to_localtime(start)),
-            'end': format_time(to_localtime(end)),
+            'start': OutputTimestamp(start),
+            'end': OutputTimestamp(end),
             'duration': (end - start).total_seconds(),
             'stats': stats.as_dict(),
             'limits': {
@@ -411,8 +411,8 @@ Number of files: {0.stats.nfiles}
 Utilization of max. archive size: {csize_max:.0%}
 '''.format(
             self,
-            start=format_time(to_localtime(self.start.replace(tzinfo=timezone.utc))),
-            end=format_time(to_localtime(self.end.replace(tzinfo=timezone.utc))),
+            start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)),
+            end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)),
             csize_max=self.cache.chunks[self.id].csize / MAX_DATA_SIZE)
 
     def __repr__(self):

src/borg/archiver.py

@@ -1334,7 +1334,7 @@ class Archiver:
         elif args.short:
             format = "{path}{NL}"
         else:
-            format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}"
+            format = "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}"
 
         def _list_inner(cache):
             archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
@@ -3122,7 +3122,7 @@ class Archiver:
                                help='only print file/directory names, nothing else')
         subparser.add_argument('--format', '--list-format', metavar='FORMAT', dest='format',
                                help='specify format for file listing '
-                                    '(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")')
+                                    '(default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}")')
         subparser.add_argument('--json', action='store_true',
                                help='Only valid for listing repository contents. Format output as JSON. '
                                     'The form of ``--format`` is ignored, '

src/borg/helpers.py

@@ -715,16 +715,19 @@ def safe_timestamp(item_timestamp_ns):
     return datetime.fromtimestamp(t_ns / 1e9)
 
 
-def format_time(t):
-    """use ISO-8601-like date and time format (human readable, with wkday and blank date/time separator)
+def format_time(ts: datetime):
     """
-    return t.strftime('%a, %Y-%m-%d %H:%M:%S')
+    Convert *ts* to a human-friendly format with textual weekday.
+    """
+    return ts.strftime('%a, %Y-%m-%d %H:%M:%S')
 
 
-def isoformat_time(t):
-    """use ISO-8601 date and time format (machine readable, no wkday, no microseconds either)
+def isoformat_time(ts: datetime):
+    """
+    Format *ts* according to ISO 8601.
     """
-    return t.strftime('%Y-%m-%dT%H:%M:%S')  # note: first make all datetime objects tz aware before adding %z here.
+    # note: first make all datetime objects tz aware before adding %z here.
+    return ts.strftime('%Y-%m-%dT%H:%M:%S.%f')
 
 
 def format_timedelta(td):
@@ -744,6 +747,24 @@ def format_timedelta(td):
     return txt
 
 
+class OutputTimestamp:
+    def __init__(self, ts: datetime):
+        if ts.tzinfo == timezone.utc:
+            ts = to_localtime(ts)
+        self.ts = ts
+
+    def __format__(self, format_spec):
+        return format_time(self.ts)
+
+    def __str__(self):
+        return '{}'.format(self)
+
+    def isoformat(self):
+        return isoformat_time(self.ts)
+
+    to_json = isoformat
+
+
 def format_file_size(v, precision=2, sign=False):
     """Format file size into a human friendly format
     """
@@ -1664,12 +1685,11 @@ class ArchiveFormatter(BaseFormatter):
         if self.json:
             self.item_data = {}
             self.format_item = self.format_item_json
-            self.format_time = self.format_time_json
         else:
             self.item_data = static_keys
 
     def format_item_json(self, item):
-        return json.dumps(self.get_item_data(item)) + '\n'
+        return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
 
     def get_item_data(self, archive_info):
         self.name = archive_info.name
@@ -1703,12 +1723,7 @@ class ArchiveFormatter(BaseFormatter):
         return self.format_time(self.archive.ts_end)
 
     def format_time(self, ts):
-        t = to_localtime(ts)
-        return format_time(t)
-
-    def format_time_json(self, ts):
-        t = to_localtime(ts)
-        return isoformat_time(t)
+        return OutputTimestamp(ts)
 
 
 class ItemFormatter(BaseFormatter):
@@ -1784,7 +1799,6 @@ class ItemFormatter(BaseFormatter):
         if self.json_lines:
             self.item_data = {}
             self.format_item = self.format_item_json
-            self.format_time = self.format_time_json
         else:
             self.item_data = static_keys
         self.format = partial_format(format, static_keys)
@@ -1796,19 +1810,19 @@ class ItemFormatter(BaseFormatter):
             'dcsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.csize),
             'num_chunks': self.calculate_num_chunks,
             'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
-            'isomtime': partial(self.format_time, 'mtime'),
-            'isoctime': partial(self.format_time, 'ctime'),
-            'isoatime': partial(self.format_time, 'atime'),
-            'mtime': partial(self.time, 'mtime'),
-            'ctime': partial(self.time, 'ctime'),
-            'atime': partial(self.time, 'atime'),
+            'isomtime': partial(self.format_iso_time, 'mtime'),
+            'isoctime': partial(self.format_iso_time, 'ctime'),
+            'isoatime': partial(self.format_iso_time, 'atime'),
+            'mtime': partial(self.format_time, 'mtime'),
+            'ctime': partial(self.format_time, 'ctime'),
+            'atime': partial(self.format_time, 'atime'),
         }
         for hash_function in hashlib.algorithms_guaranteed:
             self.add_key(hash_function, partial(self.hash_item, hash_function))
         self.used_call_keys = set(self.call_keys) & self.format_keys
 
     def format_item_json(self, item):
-        return json.dumps(self.get_item_data(item)) + '\n'
+        return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
 
     def add_key(self, key, callable_with_item):
         self.call_keys[key] = callable_with_item
@@ -1883,15 +1897,10 @@ class ItemFormatter(BaseFormatter):
         return hash.hexdigest()
 
     def format_time(self, key, item):
-        t = self.time(key, item)
-        return format_time(t)
-
-    def format_time_json(self, key, item):
-        t = self.time(key, item)
-        return isoformat_time(t)
+        return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime))
 
-    def time(self, key, item):
-        return safe_timestamp(item.get(key) or item.mtime)
+    def format_iso_time(self, key, item):
+        return self.format_time(key, item).isoformat()
 
 
 class ChunkIteratorFileWrapper:
@@ -2204,6 +2213,8 @@ class BorgJsonEncoder(json.JSONEncoder):
             return {
                 'stats': o.stats(),
             }
+        if callable(getattr(o, 'to_json', None)):
+            return o.to_json()
         return super().default(o)
 
 
@@ -2216,7 +2227,7 @@ def basic_json_data(manifest, *, cache=None, extra=None):
             'mode': key.ARG_NAME,
         },
     })
-    data['repository']['last_modified'] = isoformat_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc)))
+    data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
     if key.NAME.startswith('key file'):
         data['encryption']['keyfile'] = key.find_key()
     if cache:

src/borg/testsuite/archiver.py

@@ -60,7 +60,7 @@ from . import key
 
 src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
-ISO_FORMAT = '%Y-%m-%dT%H:%M:%S'
+ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
 
 
 def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw):
@@ -1325,6 +1325,8 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         assert isinstance(archive['duration'], float)
         assert len(archive['id']) == 64
         assert 'stats' in archive
+        assert datetime.strptime(archive['start'], ISO_FORMAT)
+        assert datetime.strptime(archive['end'], ISO_FORMAT)
 
     def test_comment(self):
         self.create_regular_file('file1', size=1024 * 80)
@@ -1787,7 +1789,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         output_warn = self.cmd('list', '--list-format', '-', test_archive)
         self.assert_in('--list-format" has been deprecated.', output_warn)
         output_1 = self.cmd('list', test_archive)
-        output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
+        output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}', test_archive)
         output_3 = self.cmd('list', '--format', '{mtime:%s} {path}{NL}', test_archive)
         self.assertEqual(output_1, output_2)
         self.assertNotEqual(output_1, output_3)
@@ -1861,7 +1863,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         file1 = items[1]
         assert file1['path'] == 'input/file1'
         assert file1['size'] == 81920
-        assert datetime.strptime(file1['isomtime'], ISO_FORMAT)  # must not raise
+        assert datetime.strptime(file1['mtime'], ISO_FORMAT)  # must not raise
 
         list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test')
         items = [json.loads(s) for s in list_archive.splitlines()]