move JSON generation and utilities to helpers

src/borg/archiver.py

@@ -52,6 +52,7 @@ from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
 from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm
 from .helpers import ErrorIgnoringTextIOWrapper
 from .helpers import ProgressIndicatorPercent
+from .helpers import BorgJsonEncoder, basic_json_data, json_print
 from .item import Item
 from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
 from .keymanager import KeyManager
@@ -65,27 +66,6 @@ from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
 STATS_HEADER = "                       Original size      Compressed size    Deduplicated size"
 
 
-class BorgJsonEncoder(json.JSONEncoder):
-    def default(self, o):
-        if isinstance(o, Repository) or isinstance(o, RemoteRepository):
-            return {
-                'id': bin_to_hex(o.id),
-                'location': o._location.canonical_path(),
-            }
-        if isinstance(o, Archive):
-            return o.info()
-        if isinstance(o, Cache):
-            return {
-                'path': o.path,
-                'stats': o.stats(),
-            }
-        return super().default(o)
-
-
-def print_as_json(obj):
-    print(json.dumps(obj, sort_keys=True, indent=4, cls=BorgJsonEncoder))
-
-
 def argument(args, str_or_bool):
     """If bool is passed, return it. If str is passed, retrieve named attribute from args."""
     if isinstance(str_or_bool, str):
@@ -385,13 +365,12 @@ class Archiver:
                 archive.save(comment=args.comment, timestamp=args.timestamp)
                 if args.progress:
                     archive.stats.show_progress(final=True)
+                args.stats |= args.json
                 if args.stats:
                     if args.json:
-                        print_as_json({
-                            'repository': repository,
-                            'cache': cache,
+                        json_print(basic_json_data(manifest, cache=cache, extra={
                             'archive': archive,
-                        })
+                        }))
                     else:
                         log_multi(DASHES,
                                   str(archive),
@@ -988,10 +967,9 @@ class Archiver:
                 write(safe_encode(formatter.format_item(archive_info)))
 
         if args.json:
-            print_as_json({
-                'repository': manifest.repository,
-                'archives': output_data,
-            })
+            json_print(basic_json_data(manifest, extra={
+                'archives': output_data
+            }))
 
         return self.exit_code
 
@@ -1001,7 +979,7 @@ class Archiver:
         if any((args.location.archive, args.first, args.last, args.prefix)):
             return self._info_archives(args, repository, manifest, key, cache)
         else:
-            return self._info_repository(args, repository, key, cache)
+            return self._info_repository(args, repository, manifest, key, cache)
 
     def _info_archives(self, args, repository, manifest, key, cache):
         def format_cmdline(cmdline):
@@ -1044,20 +1022,18 @@ class Archiver:
                 print()
 
         if args.json:
-            print_as_json({
-                'repository': repository,
-                'cache': cache,
+            json_print(basic_json_data(manifest, cache=cache, extra={
                 'archives': output_data,
-            })
+            }))
         return self.exit_code
 
-    def _info_repository(self, args, repository, key, cache):
+    def _info_repository(self, args, repository, manifest, key, cache):
+        info = basic_json_data(manifest, cache=cache, extra={
+            'security_dir': cache.security_manager.dir,
+        })
+
         if args.json:
-            encryption = {
-                'mode': key.NAME,
-            }
-            if key.NAME.startswith('key file'):
-                encryption['keyfile'] = key.find_key()
+            json_print(info)
         else:
             encryption = 'Encrypted: '
             if key.NAME == 'plaintext':
@@ -1066,18 +1042,8 @@ class Archiver:
                 encryption += 'Yes (%s)' % key.NAME
             if key.NAME.startswith('key file'):
                 encryption += '\nKey file: %s' % key.find_key()
+            info['encryption'] = encryption
 
-        info = {
-            'repository': repository,
-            'cache': cache,
-            'security_dir': cache.security_manager.dir,
-            'encryption': encryption,
-        }
-
-        if args.json:
-            info['cache_stats'] = cache.stats()
-            print_as_json(info)
-        else:
             print(textwrap.dedent("""
             Repository ID: {id}
             Location: {location}
@@ -2226,7 +2192,7 @@ class Archiver:
         subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS',
                                help='only display items with the given status characters')
         subparser.add_argument('--json', action='store_true',
-                               help='output stats as JSON')
+                               help='output stats as JSON (implies --stats)')
 
         exclude_group = subparser.add_argument_group('Exclusion options')
         exclude_group.add_argument('-e', '--exclude', dest='patterns',

src/borg/helpers.py

@@ -207,6 +207,10 @@ class Manifest:
     def id_str(self):
         return bin_to_hex(self.id)
 
+    @property
+    def last_timestamp(self):
+        return datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f")
+
     @classmethod
     def load(cls, repository, key=None, force_tam_not_required=False):
         from .item import ManifestItem
@@ -251,7 +255,7 @@ class Manifest:
         if self.timestamp is None:
             self.timestamp = datetime.utcnow().isoformat()
         else:
-            prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f")
+            prev_ts = self.last_timestamp
             incremented = (prev_ts + timedelta(microseconds=1)).isoformat()
             self.timestamp = max(incremented, datetime.utcnow().isoformat())
         manifest = ManifestItem(
@@ -1656,14 +1660,13 @@ class ItemFormatter(BaseFormatter):
             self.item_data = static_keys
 
     def begin(self):
-        from borg.archiver import BorgJsonEncoder
         if not self.json:
             return ''
-        return textwrap.dedent("""
-        {{
-            "repository": {repository},
-            "files": [
-        """).strip().format(repository=BorgJsonEncoder().encode(self.archive.repository))
+        begin = json_dump(basic_json_data(self.archive.manifest))
+        begin, _, _ = begin.rpartition('\n}')  # remove last closing brace, we want to extend the object
+        begin += ',\n'
+        begin += '    "files": [\n'
+        return begin
 
     def end(self):
         if not self.json:
@@ -2090,3 +2093,50 @@ def swidth_slice(string, max_width):
     if reverse:
         result.reverse()
     return ''.join(result)
+
+
+class BorgJsonEncoder(json.JSONEncoder):
+    def default(self, o):
+        from .repository import Repository
+        from .remote import RemoteRepository
+        from .archive import Archive
+        from .cache import Cache
+        if isinstance(o, Repository) or isinstance(o, RemoteRepository):
+            return {
+                'id': bin_to_hex(o.id),
+                'location': o._location.canonical_path(),
+            }
+        if isinstance(o, Archive):
+            return o.info()
+        if isinstance(o, Cache):
+            return {
+                'path': o.path,
+                'stats': o.stats(),
+            }
+        return super().default(o)
+
+
+def basic_json_data(manifest, *, cache=None, extra=None):
+    key = manifest.key
+    data = extra or {}
+    data.update({
+        'repository': BorgJsonEncoder().default(manifest.repository),
+        'encryption': {
+            'mode': key.NAME,
+        },
+    })
+    data['repository']['last_modified'] = format_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc)))
+    if key.NAME.startswith('key file'):
+        data['encryption']['keyfile'] = key.find_key()
+    if cache:
+        data['cache'] = cache
+    return data
+
+
+def json_dump(obj):
+    """Dump using BorgJSONEncoder."""
+    return json.dumps(obj, sort_keys=True, indent=4, cls=BorgJsonEncoder)
+
+
+def json_print(obj):
+    print(json_dump(obj))

src/borg/testsuite/archiver.py

@@ -1117,10 +1117,27 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.cmd('create', self.repository_location + '::test', 'input')
         info_repo = json.loads(self.cmd('info', '--json', self.repository_location))
-        assert len(info_repo['id']) == 64
+        repository = info_repo['repository']
+        assert len(repository['id']) == 64
+        assert 'last_modified' in repository
         assert info_repo['encryption']['mode'] == 'repokey'
         assert 'keyfile' not in info_repo['encryption']
-        assert 'cache-stats' in info_repo
+        cache = info_repo['cache']
+        stats = cache['stats']
+        assert all(isinstance(o, int) for o in stats.values())
+        assert all(key in stats for key in ('total_chunks', 'total_csize', 'total_size', 'total_unique_chunks', 'unique_csize', 'unique_size'))
+
+        info_archive = json.loads(self.cmd('info', '--json', self.repository_location + '::test'))
+        assert info_repo['repository'] == info_archive['repository']
+        assert info_repo['cache'] == info_archive['cache']
+        archives = info_archive['archives']
+        assert len(archives) == 1
+        archive = archives[0]
+        assert archive['name'] == 'test'
+        assert isinstance(archive['command_line'], list)
+        assert isinstance(archive['duration'], float)
+        assert len(archive['id']) == 64
+        assert 'stats' in archive
 
     def test_comment(self):
         self.create_regular_file('file1', size=1024 * 80)
@@ -1273,6 +1290,23 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         if has_lchflags:
             self.assert_in("x input/file3", output)
 
+    def test_create_json(self):
+        self.create_regular_file('file1', size=1024 * 80)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        create_info = json.loads(self.cmd('create', '--json', self.repository_location + '::test', 'input'))
+        # The usual keys
+        assert 'encryption' in create_info
+        assert 'repository' in create_info
+        assert 'cache' in create_info
+        assert 'last_modified' in create_info['repository']
+
+        archive = create_info['archive']
+        assert archive['name'] == 'test'
+        assert isinstance(archive['command_line'], list)
+        assert isinstance(archive['duration'], float)
+        assert len(archive['id']) == 64
+        assert 'stats' in archive
+
     def test_create_topical(self):
         now = time.time()
         self.create_regular_file('file1', size=1024 * 80)
@@ -1457,6 +1491,33 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         assert int(dsize) <= int(size)
         assert int(dcsize) <= int(csize)
 
+    def test_list_json(self):
+        self.create_regular_file('file1', size=1024 * 80)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', self.repository_location + '::test', 'input')
+        list_repo = json.loads(self.cmd('list', '--json', self.repository_location))
+        repository = list_repo['repository']
+        assert len(repository['id']) == 64
+        assert 'last_modified' in repository
+        assert list_repo['encryption']['mode'] == 'repokey'
+        assert 'keyfile' not in list_repo['encryption']
+
+        list_archive = json.loads(self.cmd('list', '--json', self.repository_location + '::test'))
+        assert list_repo['repository'] == list_archive['repository']
+        files = list_archive['files']
+        assert len(files) == 2
+        file1 = files[1]
+        assert file1['path'] == 'input/file1'
+        assert file1['size'] == 81920
+
+        list_archive = json.loads(self.cmd('list', '--json', '--format={sha256}', self.repository_location + '::test'))
+        assert list_repo['repository'] == list_archive['repository']
+        files = list_archive['files']
+        assert len(files) == 2
+        file1 = files[1]
+        assert file1['path'] == 'input/file1'
+        assert file1['sha256'] == 'b2915eb69f260d8d3c25249195f2c8f4f716ea82ec760ae929732c0262442b2b'
+
     def _get_sizes(self, compression, compressible, size=10000):
         if compressible:
             contents = b'X' * size