Add a "file_list_format" option for setting the "list" action's output format and an "archive_list_format" option for setting the "repo-list" action's format (#1179).

NEWS

@@ -1,4 +1,6 @@
 2.0.13.dev0
+ * #1179: Add a "file_list_format" option for setting the "list" action's output format and an
+   "archive_list_format" option for setting the "repo-list" action's format.
  * #1192: Fix for over-aggressive deduplication of source directories that contain the borgmatic
    runtime directory, potentially resulting in data loss (data not getting backed up) when
    snapshotting these source directories.

borgmatic/borg/list.py

@@ -18,6 +18,7 @@ MAKE_FLAGS_EXCLUDES = (
     'archive',
     'paths',
     'find_paths',
+    'format',
     *ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST,
 )
 
@@ -54,6 +55,7 @@ def make_list_command(
         + flags.make_flags('umask', config.get('umask'))
         + flags.make_flags('log-json', config.get('log_json'))
         + flags.make_flags('lock-wait', config.get('lock_wait'))
+        + flags.make_flags('format', list_arguments.format or config.get('file_list_format'))
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + (

borgmatic/borg/repo_list.py

@@ -110,7 +110,7 @@ def get_latest_archive(
     return latest_archive
 
 
-MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
+MAKE_FLAGS_EXCLUDES = ('repository', 'format', 'prefix', 'match_archives')
 
 
 def make_repo_list_command(
@@ -170,6 +170,9 @@ def make_repo_list_command(
                 )
             )
         )
+        + flags.make_flags(
+            'format', repo_list_arguments.format or config.get('archive_list_format')
+        )
         + flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)

borgmatic/commands/arguments.py

@@ -297,7 +297,14 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
     )
 
 
-OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'}
+OMITTED_FLAG_NAMES = {
+    'match-archives',
+    'progress',
+    'statistics',
+    'list-details',
+    'file-list-format',
+    'archive-list-format',
+}
 
 
 def make_argument_description(schema, flag_name):
@@ -1541,7 +1548,7 @@ def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
         action='store_true',
         help='Output only archive names',
     )
-    repo_list_group.add_argument('--format', help='Format for archive listing')
+    repo_list_group.add_argument('--format', help='Borg format for the archive listing')
     repo_list_group.add_argument(
         '--json',
         default=False,
@@ -1644,7 +1651,7 @@ def make_parsers(schema, unparsed_arguments):  # noqa: PLR0915
         action='store_true',
         help='Output only path names',
     )
-    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument('--format', help='Borg format for the file listing')
     list_group.add_argument(
         '--json',
         default=False,

borgmatic/config/schema.yaml

@@ -492,6 +492,27 @@ properties:
             If match_archives is not specified, borgmatic defaults to deriving
             the match_archives value from archive_name_format.
         example: "sh:{hostname}-*"
+    file_list_format:
+        type: string
+        description: |
+            Borg format for the files listing of the "list" action. Corresponds
+            to the "--format" flag. Defaults to "{mode} {user:6} {group:6}
+            {size:8} {mtime} {path}{extra}{NL}". With "--json", the form of the
+            format is ignored, but the keys used in it are added to the JSON
+            output. See
+            https://borgbackup.readthedocs.io/en/stable/usage/list.html for
+            details.
+        example: "{path}{extra}{NL}"
+    archive_list_format:
+        type: string
+        description: |
+            Borg format for the archives listing of the "repo-list" action.
+            Corresponds to the "--format" flag. Defaults to "{archive:<36}
+            {time} [{id}]{NL}". With "--json", the form of the format is
+            ignored, but the keys used in it are added to the JSON output. See
+            https://borgbackup.readthedocs.io/en/stable/usage/list.html for
+            details.
+        example: "{archive}{NL}"
     relocated_repo_access_is_ok:
         type: boolean
         description: |
@@ -998,6 +1019,8 @@ properties:
         type: string
         description: |
             Python format string used for log messages written to the log file.
+            See https://docs.python.org/3/library/logging.html (and specifically
+            the LogRecord attributes with "{}-formatting") for details.
         example: "[{asctime}] {levelname}: {prefix}{message}"
     monitoring_verbosity:
         type: integer

tests/unit/borg/test_list.py

@@ -19,7 +19,7 @@ def test_make_list_command_includes_log_info():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -36,7 +36,7 @@ def test_make_list_command_includes_json_but_not_info():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=True),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=True),
         global_arguments=flexmock(),
     )
 
@@ -53,7 +53,7 @@ def test_make_list_command_includes_log_debug():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -70,7 +70,7 @@ def test_make_list_command_includes_json_but_not_debug():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=True),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=True),
         global_arguments=flexmock(),
     )
 
@@ -86,7 +86,7 @@ def test_make_list_command_includes_json():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=True),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=True),
         global_arguments=flexmock(),
     )
 
@@ -96,7 +96,7 @@ def test_make_list_command_includes_json():
 def test_make_list_command_includes_log_json():
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
         ('--log-json',),
-    )
+    ).and_return(())
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
 
@@ -104,7 +104,7 @@ def test_make_list_command_includes_log_json():
         repository_path='repo',
         config={'log_json': True},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -114,7 +114,7 @@ def test_make_list_command_includes_log_json():
 def test_make_list_command_includes_lock_wait():
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
         ('--lock-wait', '5'),
-    )
+    ).and_return(())
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
 
@@ -122,13 +122,31 @@ def test_make_list_command_includes_lock_wait():
         repository_path='repo',
         config={'lock_wait': 5},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
     assert command == ('borg', 'list', '--lock-wait', '5', 'repo')
 
 
+def test_make_list_command_includes_format():
+    flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
+        ()
+    ).and_return(('--format', 'stuff'))
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+
+    command = module.make_list_command(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        list_arguments=flexmock(archive=None, paths=None, format='stuff', json=False),
+        global_arguments=flexmock(),
+    )
+
+    assert command == ('borg', 'list', '--format', 'stuff', 'repo')
+
+
 def test_make_list_command_includes_extra_borg_options():
     flexmock(module.flags).should_receive('make_flags').and_return(())
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
@@ -138,7 +156,7 @@ def test_make_list_command_includes_extra_borg_options():
         repository_path='repo',
         config={'extra_borg_options': {'list': '--extra "value with space"'}},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -156,7 +174,7 @@ def test_make_list_command_includes_archive():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive='archive', paths=None, json=False),
+        list_arguments=flexmock(archive='archive', paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -174,7 +192,7 @@ def test_make_list_command_includes_archive_and_path():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False),
+        list_arguments=flexmock(archive='archive', paths=['var/lib'], format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -190,7 +208,7 @@ def test_make_list_command_includes_local_path():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
         local_path='borg2',
     )
@@ -214,7 +232,7 @@ def test_make_list_command_includes_remote_path():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
         remote_path='borg2',
     )
@@ -233,7 +251,7 @@ def test_make_list_command_includes_umask():
         repository_path='repo',
         config={'umask': '077'},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False),
         global_arguments=flexmock(),
     )
 
@@ -249,7 +267,7 @@ def test_make_list_command_includes_short():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        list_arguments=flexmock(archive=None, paths=None, json=False, short=True),
+        list_arguments=flexmock(archive=None, paths=None, format=None, json=False, short=True),
         global_arguments=flexmock(),
     )
 
@@ -284,9 +302,9 @@ def test_make_list_command_includes_additional_flags(argument_name):
         list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             find_paths=None,
-            format=None,
             **{argument_name: 'value'},
         ),
         global_arguments=flexmock(),
@@ -344,6 +362,7 @@ def test_list_archive_calls_borg_with_flags():
     list_arguments = argparse.Namespace(
         archive='archive',
         paths=None,
+        format=None,
         json=False,
         find_paths=None,
         prefix=None,
@@ -410,6 +429,7 @@ def test_list_archive_calls_borg_with_local_path():
     list_arguments = argparse.Namespace(
         archive='archive',
         paths=None,
+        format=None,
         json=False,
         find_paths=None,
         prefix=None,

tests/unit/borg/test_repo_list.py

@@ -600,6 +600,7 @@ def test_make_repo_list_command_includes_log_info():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -629,6 +630,7 @@ def test_make_repo_list_command_includes_json_but_not_info():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=True,
             prefix=None,
             match_archives=None,
@@ -658,6 +660,7 @@ def test_make_repo_list_command_includes_log_debug():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -687,6 +690,7 @@ def test_make_repo_list_command_includes_json_but_not_debug():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=True,
             prefix=None,
             match_archives=None,
@@ -715,6 +719,7 @@ def test_make_repo_list_command_includes_json():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=True,
             prefix=None,
             match_archives=None,
@@ -729,7 +734,7 @@ def test_make_repo_list_command_includes_log_json():
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(
         ('--log-json',),
-    ).and_return(())
+    ).and_return(()).and_return(())
     flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
         None,
         None,
@@ -745,6 +750,7 @@ def test_make_repo_list_command_includes_log_json():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -759,7 +765,7 @@ def test_make_repo_list_command_includes_lock_wait():
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(
         ('--lock-wait', '5'),
-    ).and_return(())
+    ).and_return(()).and_return(())
     flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
         None,
         None,
@@ -775,6 +781,7 @@ def test_make_repo_list_command_includes_lock_wait():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -803,6 +810,7 @@ def test_make_repo_list_command_includes_list_extra_borg_options():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -831,6 +839,7 @@ def test_make_repo_list_command_with_feature_available_includes_repo_list_extra_
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -859,6 +868,7 @@ def test_make_repo_list_command_includes_local_path():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -890,6 +900,7 @@ def test_make_repo_list_command_includes_remote_path():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -921,6 +932,7 @@ def test_make_repo_list_command_includes_umask():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -935,7 +947,7 @@ def test_make_repo_list_command_transforms_prefix_into_match_archives():
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
         ('--match-archives', 'sh:foo*'),
-    )
+    ).and_return(())
     flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
         None,
         None,
@@ -948,7 +960,9 @@ def test_make_repo_list_command_transforms_prefix_into_match_archives():
         repository_path='repo',
         config={},
         local_borg_version='1.2.3',
-        repo_list_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'),
+        repo_list_arguments=flexmock(
+            archive=None, paths=None, format=None, json=False, prefix='foo'
+        ),
         global_arguments=flexmock(),
     )
 
@@ -959,7 +973,7 @@ def test_make_repo_list_command_prefers_prefix_over_archive_name_format():
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
         ('--match-archives', 'sh:foo*'),
-    )
+    ).and_return(())
     flexmock(module.flags).should_receive('make_match_archives_flags').never()
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
@@ -968,7 +982,9 @@ def test_make_repo_list_command_prefers_prefix_over_archive_name_format():
         repository_path='repo',
         config={'archive_name_format': 'bar-{now}'},
         local_borg_version='1.2.3',
-        repo_list_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'),
+        repo_list_arguments=flexmock(
+            archive=None, paths=None, format=None, json=False, prefix='foo'
+        ),
         global_arguments=flexmock(),
     )
 
@@ -993,6 +1009,7 @@ def test_make_repo_list_command_transforms_archive_name_format_into_match_archiv
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -1003,6 +1020,38 @@ def test_make_repo_list_command_transforms_archive_name_format_into_match_archiv
     assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo')
 
 
+def test_make_repo_list_command_includes_format_from_command_line():
+    flexmock(module.feature).should_receive('available').and_return(False)
+    flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(()).and_return(
+        ('--format', 'stuff')
+    )
+    flexmock(module.flags).should_receive('make_match_archives_flags').with_args(
+        None,
+        None,
+        '1.2.3',
+    ).and_return(())
+    flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(())
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',))
+
+    command = module.make_repo_list_command(
+        repository_path='repo',
+        config={},
+        local_borg_version='1.2.3',
+        repo_list_arguments=flexmock(
+            archive=None,
+            paths=None,
+            format='stuff',
+            json=False,
+            prefix=None,
+            match_archives=None,
+            short=False,
+        ),
+        global_arguments=flexmock(),
+    )
+
+    assert command == ('borg', 'list', '--format', 'stuff', 'repo')
+
+
 def test_make_repo_list_command_includes_short():
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.flags).should_receive('make_flags').and_return(())
@@ -1021,6 +1070,7 @@ def test_make_repo_list_command_includes_short():
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
@@ -1064,11 +1114,11 @@ def test_make_repo_list_command_includes_additional_flags(argument_name):
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,
             find_paths=None,
-            format=None,
             **{argument_name: 'value'},
         ),
         global_arguments=flexmock(),
@@ -1100,11 +1150,11 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives='foo-*',
             find_paths=None,
-            format=None,
         ),
         global_arguments=flexmock(),
     )
@@ -1173,6 +1223,7 @@ def test_make_repo_list_command_with_date_based_matching_calls_borg_with_date_ba
         repo_list_arguments=flexmock(
             archive=None,
             paths=None,
+            format=None,
             json=False,
             prefix=None,
             match_archives=None,