support common options

docs/man_intro.rst

@@ -14,7 +14,7 @@ deduplicating and encrypting backup tool
 SYNOPSIS
 --------
 
-borg <command> [options] [arguments]
+borg [common options] <command> [options] [arguments]
 
 DESCRIPTION
 -----------

setup.py

@@ -259,7 +259,7 @@ class build_usage(Command):
                               "command_": command.replace(' ', '_'),
                               "underline": '-' * len('borg ' + command)}
                     doc.write(".. _borg_{command_}:\n\n".format(**params))
-                    doc.write("borg {command}\n{underline}\n::\n\n    borg {command}".format(**params))
+                    doc.write("borg {command}\n{underline}\n::\n\n    borg [common options] {command}".format(**params))
                     self.write_usage(parser, doc)
                     epilog = parser.epilog
                     parser.epilog = None
@@ -402,10 +402,10 @@ class build_man(Command):
             if is_intermediary:
                 subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
                 for subcommand in subparsers.choices:
-                    write('| borg', command, subcommand, '...')
+                    write('| borg', '[common options]', command, subcommand, '...')
                     self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand))
             else:
-                write('borg', command, end='')
+                write('borg', '[common options]', command, end='')
                 self.write_usage(write, parser)
             write('\n')
 

src/borg/archiver.py

@@ -1881,58 +1881,166 @@ class Archiver:
                 epilog = [line for line in epilog if not line.startswith('.. man')]
             return '\n'.join(epilog)
 
-        common_parser = argparse.ArgumentParser(add_help=False, prog=self.prog)
+        class CommonOptions:
+            """
+            Support class to allow specifying common options directly after the top-level command.
+
+            Normally options can only be specified on the parser defining them, which means
+            that generally speaking *all* options go after all sub-commands. This is annoying
+            for common options in scripts, e.g. --remote-path or logging options.
+
+            This class allows adding the same set of options to both the top-level parser
+            and the final sub-command parsers (but not intermediary sub-commands, at least for now).
+
+            It does so by giving every option's target name ("dest") a suffix indicating it's level
+            -- no two options in the parser hierarchy can have the same target --
+            then, after parsing the command line, multiple definitions are resolved.
+
+            Defaults are handled by only setting them on the top-level parser and setting
+            a sentinel object in all sub-parsers, which then allows to discern which parser
+            supplied the option.
+            """
 
-        common_group = common_parser.add_argument_group('Common options')
-        common_group.add_argument('-h', '--help', action='help', help='show this help message and exit')
-        common_group.add_argument('--critical', dest='log_level',
-                                  action='store_const', const='critical', default='warning',
-                                  help='work on log level CRITICAL')
-        common_group.add_argument('--error', dest='log_level',
-                                  action='store_const', const='error', default='warning',
-                                  help='work on log level ERROR')
-        common_group.add_argument('--warning', dest='log_level',
-                                  action='store_const', const='warning', default='warning',
-                                  help='work on log level WARNING (default)')
-        common_group.add_argument('--info', '-v', '--verbose', dest='log_level',
-                                  action='store_const', const='info', default='warning',
-                                  help='work on log level INFO')
-        common_group.add_argument('--debug', dest='log_level',
-                                  action='store_const', const='debug', default='warning',
-                                  help='enable debug output, work on log level DEBUG')
-        common_group.add_argument('--debug-topic', dest='debug_topics',
-                                  action='append', metavar='TOPIC', default=[],
-                                  help='enable TOPIC debugging (can be specified multiple times). '
-                                       'The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.')
-        common_group.add_argument('-p', '--progress', dest='progress', action='store_true',
-                                  help='show progress information')
-        common_group.add_argument('--log-json', dest='log_json', action='store_true',
-                                  help='Output one JSON object per log line instead of formatted text.')
-        common_group.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1,
-                                  help='wait for the lock, but max. N seconds (default: %(default)d).')
-        common_group.add_argument('--show-version', dest='show_version', action='store_true', default=False,
-                                  help='show/log the borg version')
-        common_group.add_argument('--show-rc', dest='show_rc', action='store_true', default=False,
-                                  help='show/log the return code (rc)')
-        common_group.add_argument('--no-files-cache', dest='cache_files', action='store_false',
-                                  help='do not load/update the file metadata cache used to detect unchanged files')
-        common_group.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, metavar='M',
-                                  help='set umask to M (local and remote, default: %(default)04o)')
-        common_group.add_argument('--remote-path', dest='remote_path', metavar='PATH',
-                                  help='use PATH as borg executable on the remote (default: "borg")')
-        common_group.add_argument('--remote-ratelimit', dest='remote_ratelimit', type=int, metavar='rate',
-                                  help='set remote network upload rate limit in kiByte/s (default: 0=unlimited)')
-        common_group.add_argument('--consider-part-files', dest='consider_part_files',
-                                  action='store_true', default=False,
-                                  help='treat part files like normal files (e.g. to list/extract them)')
-
-        parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups')
+            # From lowest precedence to highest precedence:
+            # An option specified on the parser belonging to index 0 is overridden if the
+            # same option is specified on any parser with a higher index.
+            SUFFIX_PRECEDENCE = ('_maincommand', '_subcommand')
+
+            def __init__(self):
+                from collections import defaultdict
+
+                # Maps suffixes to sets of target names.
+                # E.g. common_options["_subcommand"] = {..., "log_level", ...}
+                self.common_options = defaultdict(defaultdict)
+                self.append_options = set()
+                self.default_sentinel = object()
+
+            def add_common_group(self, parser, suffix='_subcommand', provide_defaults=False):
+                """
+                Add common options to *parser*.
+
+                *provide_defaults* must only be True exactly once in a parser hierarchy,
+                at the top level, and False on all lower levels. The default is chosen
+                accordingly.
+
+                *suffix* indicates the suffix to use internally. It also indicates
+                which precedence the *parser* has for common options. See SUFFIX_PRECEDENCE.
+                """
+                assert suffix in self.SUFFIX_PRECEDENCE
+
+                def add_argument(*args, **kwargs):
+                    if 'dest' in kwargs:
+                        is_append = kwargs.get('action') == 'append'
+                        if is_append:
+                            self.append_options.add(kwargs['dest'])
+                            assert kwargs['default'] == [], 'The default is explicitly constructed as an empty list in resolve()'
+                        else:
+                            self.common_options.setdefault(suffix, set()).add(kwargs['dest'])
+                        kwargs['dest'] += suffix
+                        if not provide_defaults and 'default' in kwargs:
+                            # Interpolate help now, in case the %(default)d (or so) is mentioned,
+                            # to avoid producing incorrect help output.
+                            # Assumption: Interpolated output can safely be interpolated again,
+                            # which should always be the case.
+                            # Note: We control all inputs.
+                            kwargs['help'] = kwargs['help'] % kwargs
+                            if not is_append:
+                                kwargs['default'] = self.default_sentinel
+
+                    common_group.add_argument(*args, **kwargs)
+
+                common_group = parser.add_argument_group('Common options')
+
+                add_argument('-h', '--help', action='help', help='show this help message and exit')
+                add_argument('--critical', dest='log_level',
+                             action='store_const', const='critical', default='warning',
+                             help='work on log level CRITICAL')
+                add_argument('--error', dest='log_level',
+                             action='store_const', const='error', default='warning',
+                             help='work on log level ERROR')
+                add_argument('--warning', dest='log_level',
+                             action='store_const', const='warning', default='warning',
+                             help='work on log level WARNING (default)')
+                add_argument('--info', '-v', '--verbose', dest='log_level',
+                             action='store_const', const='info', default='warning',
+                             help='work on log level INFO')
+                add_argument('--debug', dest='log_level',
+                             action='store_const', const='debug', default='warning',
+                             help='enable debug output, work on log level DEBUG')
+                add_argument('--debug-topic', dest='debug_topics',
+                             action='append', metavar='TOPIC', default=[],
+                             help='enable TOPIC debugging (can be specified multiple times). '
+                                  'The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.')
+                add_argument('-p', '--progress', dest='progress', action='store_true',
+                             help='show progress information')
+                add_argument('--log-json', dest='log_json', action='store_true',
+                             help='Output one JSON object per log line instead of formatted text.')
+                add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1,
+                             help='wait for the lock, but max. N seconds (default: %(default)d).')
+                add_argument('--show-version', dest='show_version', action='store_true', default=False,
+                             help='show/log the borg version')
+                add_argument('--show-rc', dest='show_rc', action='store_true', default=False,
+                             help='show/log the return code (rc)')
+                add_argument('--no-files-cache', dest='cache_files', action='store_false',
+                             help='do not load/update the file metadata cache used to detect unchanged files')
+                add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, metavar='M',
+                             help='set umask to M (local and remote, default: %(default)04o)')
+                add_argument('--remote-path', dest='remote_path', metavar='PATH',
+                             help='use PATH as borg executable on the remote (default: "borg")')
+                add_argument('--remote-ratelimit', dest='remote_ratelimit', type=int, metavar='rate',
+                             help='set remote network upload rate limit in kiByte/s (default: 0=unlimited)')
+                add_argument('--consider-part-files', dest='consider_part_files',
+                             action='store_true', default=False,
+                             help='treat part files like normal files (e.g. to list/extract them)')
+
+            def resolve(self, args: argparse.Namespace):  # Namespace has "in" but otherwise is not like a dict.
+                """
+                Resolve the multiple definitions of each common option to the final value.
+                """
+                for suffix in self.SUFFIX_PRECEDENCE:
+                    # From highest level to lowest level, so the "most-specific" option wins, e.g.
+                    # "borg --debug create --info" shall result in --info being effective.
+                    for dest in self.common_options.get(suffix, []):
+                        # map_from is this suffix' option name, e.g. log_level_subcommand
+                        # map_to is the target name, e.g. log_level
+                        map_from = dest + suffix
+                        map_to = dest
+                        # Retrieve value; depending on the action it may not exist, but usually does
+                        # (store_const/store_true/store_false), either because the action implied a default
+                        # or a default is explicitly supplied.
+                        # Note that defaults on lower levels are replaced with default_sentinel.
+                        # Only the top level has defaults.
+                        value = getattr(args, map_from, self.default_sentinel)
+                        if value is not self.default_sentinel:
+                            # value was indeed specified on this level. Transfer value to target,
+                            # and un-clobber the args (for tidiness - you *cannot* use the suffixed
+                            # names for other purposes, obviously).
+                            setattr(args, map_to, value)
+                            delattr(args, map_from)
+
+                # Options with an "append" action need some special treatment. Instead of
+                # overriding values, all specified values are merged together.
+                for dest in self.append_options:
+                    option_value = []
+                    for suffix in self.SUFFIX_PRECEDENCE:
+                        # Find values of this suffix, if any, and add them to the final list
+                        values = getattr(args, dest + suffix, [])
+                        option_value.extend(values)
+                    setattr(args, dest, option_value)
+
+        parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups',
+                                         add_help=False)
+        parser.common_options = CommonOptions()
         parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__,
                             help='show version number and exit')
-        subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
+        parser.common_options.add_common_group(parser, '_maincommand', provide_defaults=True)
 
+        common_parser = argparse.ArgumentParser(add_help=False, prog=self.prog)
         # some empty defaults for all subparsers
         common_parser.set_defaults(paths=[], patterns=[])
+        parser.common_options.add_common_group(common_parser, '_subcommand')
+
+        subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
 
         serve_epilog = process_epilog("""
         This command starts a repository server process. This command is usually not used manually.
@@ -3358,6 +3466,7 @@ class Archiver:
             args = self.preprocess_args(args)
         parser = self.build_parser()
         args = parser.parse_args(args or ['-h'])
+        parser.common_options.resolve(args)
         # This works around http://bugs.python.org/issue9351
         func = getattr(args, 'func', None) or getattr(args, 'fallback_func')
         if func == self.do_create and not args.paths:

src/borg/testsuite/archiver.py

@@ -1680,6 +1680,12 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         assert log_message['name'].startswith('borg.')
         assert isinstance(log_message['message'], str)
 
+    def test_common_options(self):
+        self.create_test_files()
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        log = self.cmd('--debug', 'create', self.repository_location + '::test', 'input')
+        assert 'security: read previous_location' in log
+
     def _get_sizes(self, compression, compressible, size=10000):
         if compressible:
             contents = b'X' * size