Support setting whole lists and dicts from the command-line (#303).

borgmatic/commands/arguments.py

@@ -1,11 +1,16 @@
 import collections
+import decimal
 import itertools
+import io
 import json
 import re
 import sys
 from argparse import ArgumentParser
 
 from borgmatic.config import collect
+import borgmatic.config.schema
+
+import ruamel.yaml
 
 ACTION_ALIASES = {
     'repo-create': ['rcreate', 'init', '-I'],
@@ -308,8 +313,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
 
         --foo.bar
 
-    If "foo" is instead an array of objects, it will get added like this
+    If "foo" is instead an array of objects, both of the following will get added:
 
+        --foo
         --foo[0].bar
 
     And if names are also passed in, they are considered to be the name components of an option
@@ -328,12 +334,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
             raise ValueError(f'Unknown type in configuration schema: {schema_type}')
 
     # If this is an "object" type, recurse for each child option ("property").
-    if schema_type in {'object', 'array'}:
-        properties = (
-            schema.get('items', {}).get('properties')
-            if schema_type == 'array'
-            else schema.get('properties')
-        )
+    if schema_type == 'object':
+        properties = schema.get('properties')
 
         if properties:
             for name, child in properties.items():
@@ -341,23 +343,37 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
                     arguments_group,
                     child,
                     unparsed_arguments,
-                    names + ((name + '[0]',) if child.get('type') == 'array' else (name,)),
+                    names + (name,)
                 )
 
         return
 
+    # If this is an "array" type, recurse for each child option of its items type. Don't return yet,
+    # so that a flag also gets added below for the array itself.
+    if schema_type == 'array':
+        properties = borgmatic.config.schema.get_properties(schema.get('items', {}))
+
+        if properties:
+            for name, child in properties.items():
+                add_arguments_from_schema(
+                    arguments_group,
+                    child,
+                    unparsed_arguments,
+                    names[:-1] + (f'{names[-1]}[0]',) + (name,)
+                )
+
     flag_name = '.'.join(names)
     description = schema.get('description')
     metavar = names[-1].upper()
 
-    if schema_type == 'array':
-        metavar = metavar.rstrip('S')
-
     if description:
         if schema_type == 'array':
-            items_schema = schema.get('items', {})
+            example_buffer = io.StringIO()
+            yaml = ruamel.yaml.YAML(typ='safe')
+            yaml.default_flow_style = True
+            yaml.dump(schema.get('example'), example_buffer)
 
-            description += ' Can specify flag multiple times.'
+            description += f' Example value: "{example_buffer.getvalue().strip()}"'
 
         if '[0]' in flag_name:
             description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).'
@@ -365,7 +381,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         description = description.replace('%', '%%')
 
     try:
-        argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type]
+        argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type]
     except KeyError:
         raise ValueError(f'Unknown type in configuration schema: {schema_type}')
 
@@ -373,7 +389,6 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
         f"--{flag_name.replace('_', '-')}",
         type=argument_type,
         metavar=metavar,
-        action='append' if schema_type == 'array' else None,
         help=description,
     )
 

borgmatic/config/arguments.py

@@ -21,7 +21,7 @@ def set_values(config, keys, value):
         ('mylist[0]', 'foo')
 
     This looks for the zeroth element of "mylist" in the given configuration. And within that value,
-    it looks up "foo" and sets it to the given value. Finally:
+    it looks up "foo" and sets it to the given value.
     '''
     if not keys:
         return
@@ -89,6 +89,22 @@ def type_for_option(schema, option_keys):
         return None
 
 
+def convert_value_type(value, option_type):
+    '''
+    Given a string value and its schema type as a string, determine its logical type (string,
+    boolean, integer, etc.), and return it converted to that type.
+
+    If the option type is a string, leave the value as a string so that special characters in it
+    don't get interpreted as YAML during conversion.
+
+    Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
+    '''
+    if option_type == 'string':
+        return value
+
+    return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
+
+
 def prepare_arguments_for_config(global_arguments, schema):
     '''
     Given global arguments as an argparse.Namespace and a configuration schema dict, parse each
@@ -125,11 +141,11 @@ def prepare_arguments_for_config(global_arguments, schema):
             prepared_values.append(
                 (
                     keys,
-                    value,
+                    convert_value_type(value, option_type),
                 )
             )
         except ruamel.yaml.error.YAMLError as error:
-            raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
+            raise ValueError(f'Invalid override "{argument_name}": {error.problem}')
 
     return tuple(prepared_values)
 

borgmatic/config/generate.py

@@ -1,12 +1,12 @@
 import collections
 import io
-import itertools
 import os
 import re
 
 import ruamel.yaml
 
 from borgmatic.config import load, normalize
+import borgmatic.config.schema
 
 INDENT = 4
 SEQUENCE_INDENT = 2
@@ -22,27 +22,6 @@ def insert_newline_before_comment(config, field_name):
     )
 
 
-def get_properties(schema):
-    '''
-    Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
-    potential properties, returned their merged properties instead (interleaved so the first
-    properties of each sub-schema come first). The idea is that the user should see all possible
-    options even if they're not all possible together.
-    '''
-    if 'oneOf' in schema:
-        return dict(
-            item
-            for item in itertools.chain(
-                *itertools.zip_longest(
-                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
-                )
-            )
-            if item is not None
-        )
-
-    return schema['properties']
-
-
 def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema and a source configuration, generate and return sample
@@ -78,7 +57,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
                         sub_schema, (source_config or {}).get(field_name, {}), level + 1
                     ),
                 )
-                for field_name, sub_schema in get_properties(schema).items()
+                for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items()
             ]
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
@@ -189,7 +168,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
         return
 
     for field_name in config[0].keys():
-        field_schema = get_properties(schema['items']).get(field_name, {})
+        field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {})
         description = field_schema.get('description')
 
         # No description to use? Skip it.
@@ -223,7 +202,7 @@ def add_comments_to_configuration_object(
         if skip_first and index == 0:
             continue
 
-        field_schema = get_properties(schema).get(field_name, {})
+        field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {})
         description = field_schema.get('description', '').strip()
 
         # If this isn't a default key, add an indicator to the comment flagging it to be commented

borgmatic/config/normalize.py

@@ -326,7 +326,11 @@ def normalize(config_filename, config):
         config['repositories'] = []
 
         for repository_dict in repositories:
-            repository_path = repository_dict['path']
+            repository_path = repository_dict.get('path')
+
+            if repository_path is None:
+                continue
+
             if '~' in repository_path:
                 logs.append(
                     logging.makeLogRecord(

borgmatic/config/schema.py

@@ -0,0 +1,22 @@
+import itertools
+
+
+def get_properties(schema):
+    '''
+    Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
+    potential properties, returned their merged properties instead (interleaved so the first
+    properties of each sub-schema come first). The idea is that the user should see all possible
+    options even if they're not all possible together.
+    '''
+    if 'oneOf' in schema:
+        return dict(
+            item
+            for item in itertools.chain(
+                *itertools.zip_longest(
+                    *[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
+                )
+            )
+            if item is not None
+        )
+
+    return schema.get('properties', {})

borgmatic/config/schema.yaml

@@ -53,8 +53,7 @@ properties:
             output of "borg help placeholders" for details. See ssh_command for
             SSH options like identity file or port. If systemd service is used,
             then add local repository paths in the systemd service file to the
-            ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a
-            list of plain path strings.
+            ReadWritePaths list.
         example:
             - path: ssh://user@backupserver/./sourcehostname.borg
               label: backupserver
@@ -738,6 +737,10 @@ properties:
             List of one or more consistency checks to run on a periodic basis
             (if "frequency" is set) or every time borgmatic runs checks (if
             "frequency" is omitted).
+        example:
+          - name: archives
+            frequency: 2 weeks
+          - name: repository
     check_repositories:
         type: array
         items:
@@ -1115,6 +1118,10 @@ properties:
             List of one or more command hooks to execute, triggered at
             particular points during borgmatic's execution. For each command
             hook, specify one of "before" or "after", not both.
+        example:
+            - before: action
+              when: [create]
+              run: [echo Backing up.]
     bootstrap:
         type: object
         properties:
@@ -1329,6 +1336,9 @@ properties:
             https://www.postgresql.org/docs/current/app-pgdump.html and
             https://www.postgresql.org/docs/current/libpq-ssl.html for
             details.
+        example:
+            - name: users
+              hostname: database.example.org
     mariadb_databases:
         type: array
         items:
@@ -1473,6 +1483,9 @@ properties:
             added to your source directories at runtime and streamed directly
             to Borg. Requires mariadb-dump/mariadb commands. See
             https://mariadb.com/kb/en/library/mysqldump/ for details.
+        example:
+            - name: users
+              hostname: database.example.org
     mysql_databases:
         type: array
         items:
@@ -1618,6 +1631,9 @@ properties:
             to Borg. Requires mysqldump/mysql commands. See
             https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for
             details.
+        example:
+            - name: users
+              hostname: database.example.org
     sqlite_databases:
         type: array
         items:
@@ -1647,6 +1663,15 @@ properties:
                         Path to the SQLite database file to restore to. Defaults
                         to the "path" option.
                     example: /var/lib/sqlite/users.db
+        description: |
+            List of one or more SQLite databases to dump before creating a
+            backup, run once per configuration file. The database dumps are
+            added to your source directories at runtime and streamed directly to
+            Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html
+            for details.
+        example:
+            - name: users
+              path: /var/lib/db.sqlite
     mongodb_databases:
         type: array
         items:
@@ -1749,6 +1774,9 @@ properties:
             to Borg. Requires mongodump/mongorestore commands. See
             https://docs.mongodb.com/database-tools/mongodump/ and
             https://docs.mongodb.com/database-tools/mongorestore/ for details.
+        example:
+            - name: users
+              hostname: database.example.org
     ntfy:
         type: object
         required: ['topic']
@@ -2231,9 +2259,13 @@ properties:
                     properties:
                         url:
                             type: string
+                            description: URL of this Apprise service.
                             example: "gotify://hostname/token"
                         label:
                             type: string
+                            description: |
+                                Label used in borgmatic logs for this Apprise
+                                service.
                             example: gotify
                 description: |
                     A list of Apprise services to publish to with URLs and

tests/unit/config/test_generate.py

@@ -4,82 +4,6 @@ from flexmock import flexmock
 from borgmatic.config import generate as module
 
 
-def test_get_properties_with_simple_object():
-    schema = {
-        'type': 'object',
-        'properties': dict(
-            [
-                ('field1', {'example': 'Example'}),
-            ]
-        ),
-    }
-
-    assert module.get_properties(schema) == schema['properties']
-
-
-def test_get_properties_merges_oneof_list_properties():
-    schema = {
-        'type': 'object',
-        'oneOf': [
-            {
-                'properties': dict(
-                    [
-                        ('field1', {'example': 'Example 1'}),
-                        ('field2', {'example': 'Example 2'}),
-                    ]
-                ),
-            },
-            {
-                'properties': dict(
-                    [
-                        ('field2', {'example': 'Example 2'}),
-                        ('field3', {'example': 'Example 3'}),
-                    ]
-                ),
-            },
-        ],
-    }
-
-    assert module.get_properties(schema) == dict(
-        schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties']
-    )
-
-
-def test_get_properties_interleaves_oneof_list_properties():
-    schema = {
-        'type': 'object',
-        'oneOf': [
-            {
-                'properties': dict(
-                    [
-                        ('field1', {'example': 'Example 1'}),
-                        ('field2', {'example': 'Example 2'}),
-                        ('field3', {'example': 'Example 3'}),
-                    ]
-                ),
-            },
-            {
-                'properties': dict(
-                    [
-                        ('field4', {'example': 'Example 4'}),
-                        ('field5', {'example': 'Example 5'}),
-                    ]
-                ),
-            },
-        ],
-    }
-
-    assert module.get_properties(schema) == dict(
-        [
-            ('field1', {'example': 'Example 1'}),
-            ('field4', {'example': 'Example 4'}),
-            ('field2', {'example': 'Example 2'}),
-            ('field5', {'example': 'Example 5'}),
-            ('field3', {'example': 'Example 3'}),
-        ]
-    )
-
-
 def test_schema_to_sample_configuration_generates_config_map_with_examples():
     schema = {
         'type': 'object',

tests/unit/config/test_schema.py

@@ -0,0 +1,80 @@
+from borgmatic.config import schema as module
+
+
+def test_get_properties_with_simple_object():
+    schema = {
+        'type': 'object',
+        'properties': dict(
+            [
+                ('field1', {'example': 'Example'}),
+            ]
+        ),
+    }
+
+    assert module.get_properties(schema) == schema['properties']
+
+
+def test_get_properties_merges_oneof_list_properties():
+    schema = {
+        'type': 'object',
+        'oneOf': [
+            {
+                'properties': dict(
+                    [
+                        ('field1', {'example': 'Example 1'}),
+                        ('field2', {'example': 'Example 2'}),
+                    ]
+                ),
+            },
+            {
+                'properties': dict(
+                    [
+                        ('field2', {'example': 'Example 2'}),
+                        ('field3', {'example': 'Example 3'}),
+                    ]
+                ),
+            },
+        ],
+    }
+
+    assert module.get_properties(schema) == dict(
+        schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties']
+    )
+
+
+def test_get_properties_interleaves_oneof_list_properties():
+    schema = {
+        'type': 'object',
+        'oneOf': [
+            {
+                'properties': dict(
+                    [
+                        ('field1', {'example': 'Example 1'}),
+                        ('field2', {'example': 'Example 2'}),
+                        ('field3', {'example': 'Example 3'}),
+                    ]
+                ),
+            },
+            {
+                'properties': dict(
+                    [
+                        ('field4', {'example': 'Example 4'}),
+                        ('field5', {'example': 'Example 5'}),
+                    ]
+                ),
+            },
+        ],
+    }
+
+    assert module.get_properties(schema) == dict(
+        [
+            ('field1', {'example': 'Example 1'}),
+            ('field4', {'example': 'Example 4'}),
+            ('field2', {'example': 'Example 2'}),
+            ('field5', {'example': 'Example 5'}),
+            ('field3', {'example': 'Example 3'}),
+        ]
+    )
+
+
+