Override particular configuration options from the command-line via "--override" flag (#268).

NEWS

@@ -1,3 +1,8 @@
+1.4.21.dev0
+ * #268: Override particular configuration options from the command-line via "--override" flag. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+
 1.4.20
  * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
  * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and

borgmatic/commands/arguments.py

@@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments):
         default=None,
         help='Write log messages to this file instead of syslog',
     )
+    global_group.add_argument(
+        '--override',
+        metavar='SECTION.OPTION=VALUE',
+        nargs='+',
+        dest='overrides',
+        help='One or more configuration file options to override with specified values',
+    )
     global_group.add_argument(
         '--version',
         dest='version',

borgmatic/commands/borgmatic.py

@@ -372,7 +372,7 @@ def run_actions(
                 yield json.loads(json_output)
 
 
-def load_configurations(config_filenames):
+def load_configurations(config_filenames, overrides=None):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -386,7 +386,7 @@ def load_configurations(config_filenames):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename()
+                config_filename, validate.schema_filename(), overrides
             )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
@@ -584,7 +584,7 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames)
+    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
 
     colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
     try:

borgmatic/config/override.py

@@ -0,0 +1,71 @@
+import io
+
+import ruamel.yaml
+
+
+def set_values(config, keys, value):
+    '''
+    Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
+    descend into the hierarchy based on the keys to set the value into the right place.
+    '''
+    if not keys:
+        return
+
+    first_key = keys[0]
+    if len(keys) == 1:
+        config[first_key] = value
+        return
+
+    if first_key not in config:
+        config[first_key] = {}
+
+    set_values(config[first_key], keys[1:], value)
+
+
+def convert_value_type(value):
+    '''
+    Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
+    converted to that type.
+    '''
+    return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
+
+
+def parse_overrides(raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value",
+    parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
+    instance, given the following raw overrides:
+
+        ['section.my_option=value1', 'section.other_option=value2']
+
+    ... return this:
+
+        (
+            (('section', 'my_option'), 'value1'),
+            (('section', 'other_option'), 'value2'),
+        )
+
+    Raise ValueError if an override can't be parsed.
+    '''
+    if not raw_overrides:
+        return ()
+
+    try:
+        return tuple(
+            (tuple(raw_keys.split('.')), convert_value_type(value))
+            for raw_override in raw_overrides
+            for raw_keys, value in (raw_override.split('=', 1),)
+        )
+    except ValueError:
+        raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
+
+
+def apply_overrides(config, raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value"
+    and a configuration dict, parse each override and set it the configuration dict.
+    '''
+    overrides = parse_overrides(raw_overrides)
+
+    for (keys, value) in overrides:
+        set_values(config, keys, value)

borgmatic/config/validate.py

@@ -6,7 +6,7 @@ import pykwalify.core
 import pykwalify.errors
 import ruamel.yaml
 
-from borgmatic.config import load
+from borgmatic.config import load, override
 
 
 def schema_filename():
@@ -82,11 +82,12 @@ def remove_examples(schema):
     return schema
 
 
-def parse_configuration(config_filename, schema_filename):
+def parse_configuration(config_filename, schema_filename, overrides=None):
     '''
-    Given the path to a config filename in YAML format and the path to a schema filename in
-    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
-    dicts and lists corresponding to the schema. Example return value:
+    Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
+    YAML schema format, a sequence of configuration file override strings in the form of
+    "section.option=value", return the parsed configuration as a data structure of nested dicts and
+    lists corresponding to the schema. Example return value:
 
        {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
        'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename):
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
+    override.apply_overrides(config, overrides)
+
     validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
     parsed_result = validator.validate(raise_exception=False)
 

docs/how-to/make-per-application-backups.md

@@ -115,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings
 directly, please see the section above about standard includes.
 
 
+## Configuration overrides
+
+In more complex multi-application setups, you may want to override particular
+borgmatic configuration file options at the time you run borgmatic. For
+instance, you could reuse a common configuration file for multiple
+applications, but then set the repository for each application at runtime. Or
+you might want to try a variant of an option for testing purposes without
+actually touching your configuration file.
+
+Whatever the reason, you can override borgmatic configuration options at the
+command-line via the `--override` flag. Here's an example:
+
+```bash
+borgmatic create --override location.remote_path=borg1
+```
+
+What this does is load your configuration files, and for each one, disregard
+the configured value for the `remote_path` option in the `location` section,
+and use the value of `borg1` instead.
+
+Note that the value is parsed as an actual YAML string, so you can even set
+list values by using brackets. For instance:
+
+```bash
+borgmatic create --override location.repositories=[test1.borg,test2.borg]
+```
+
+There is not currently a way to override a single element of a list without
+replacing the whole list.
+
+Be sure to quote your overrides if they contain spaces or other characters
+that your shell may interpret.
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.20'
+VERSION = '1.4.21.dev0'
 
 
 setup(

tests/integration/config/test_override.py

@@ -0,0 +1,40 @@
+import pytest
+
+from borgmatic.config import override as module
+
+
+@pytest.mark.parametrize(
+    'value,expected_result',
+    (
+        ('thing', 'thing'),
+        ('33', 33),
+        ('33b', '33b'),
+        ('true', True),
+        ('false', False),
+        ('[foo]', ['foo']),
+        ('[foo, bar]', ['foo', 'bar']),
+    ),
+)
+def test_convert_value_type_coerces_values(value, expected_result):
+    assert module.convert_value_type(value) == expected_result
+
+
+def test_apply_overrides_updates_config():
+    raw_overrides = [
+        'section.key=value1',
+        'other_section.thing=value2',
+        'section.nested.key=value3',
+        'new.foo=bar',
+    ]
+    config = {
+        'section': {'key': 'value', 'other': 'other_value'},
+        'other_section': {'thing': 'thing_value'},
+    }
+
+    module.apply_overrides(config, raw_overrides)
+
+    assert config == {
+        'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
+        'other_section': {'thing': 'value2'},
+        'new': {'foo': 'bar'},
+    }

tests/integration/config/test_validate.py

@@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error():
 
     with pytest.raises(module.Validation_error):
         module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_applies_overrides():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+            local_path: borg1
+        '''
+    )
+
+    result = module.parse_configuration(
+        'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2']
+    )
+
+    assert result == {
+        'location': {
+            'source_directories': ['/home'],
+            'repositories': ['hostname.borg'],
+            'local_path': 'borg2',
+        }
+    }

tests/unit/config/test_override.py

@@ -0,0 +1,82 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import override as module
+
+
+def test_set_values_with_empty_keys_bails():
+    config = {}
+
+    module.set_values(config, keys=(), value='value')
+
+    assert config == {}
+
+
+def test_set_values_with_one_key_sets_it_into_config():
+    config = {}
+
+    module.set_values(config, keys=('key',), value='value')
+
+    assert config == {'key': 'value'}
+
+
+def test_set_values_with_one_key_overwrites_existing_key():
+    config = {'key': 'old_value', 'other': 'other_value'}
+
+    module.set_values(config, keys=('key',), value='value')
+
+    assert config == {'key': 'value', 'other': 'other_value'}
+
+
+def test_set_values_with_multiple_keys_creates_hierarchy():
+    config = {}
+
+    module.set_values(config, ('section', 'key'), 'value')
+
+    assert config == {'section': {'key': 'value'}}
+
+
+def test_set_values_with_multiple_keys_updates_hierarchy():
+    config = {'section': {'other': 'other_value'}}
+    module.set_values(config, ('section', 'key'), 'value')
+
+    assert config == {'section': {'key': 'value', 'other': 'other_value'}}
+
+
+def test_parse_overrides_splits_keys_and_values():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.my_option=value1', 'section.other_option=value2']
+    expected_result = (
+        (('section', 'my_option'), 'value1'),
+        (('section', 'other_option'), 'value2'),
+    )
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_allows_value_with_equal_sign():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.option=this===value']
+    expected_result = ((('section', 'option'), 'this===value'),)
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_raises_on_missing_equal_sign():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.option']
+
+    with pytest.raises(ValueError):
+        module.parse_overrides(raw_overrides)
+
+
+def test_parse_overrides_allows_value_with_single_key():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['option=value']
+    expected_result = ((('option',), 'value'),)
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_handles_empty_overrides():
+    module.parse_overrides(raw_overrides=None) == ()