Config generation support for sequences of maps, needed for database dump hooks (#225).

.gitignore

@@ -5,6 +5,7 @@
 .coverage
 .pytest_cache
 .tox
+__pycache__
 build/
 dist/
 pip-wheel-metadata/

NEWS

@@ -1,3 +1,6 @@
+1.3.27.dev0
+ * #225: Database dump/restore hooks for PostgreSQL (incomplete as of right now).
+
 1.3.26
  * #224: Fix "borgmatic list --successful" with a slightly better heuristic for listing successful
    (non-checkpoint) archives.

borgmatic/config/convert.py

@@ -54,10 +54,10 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
     # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration(destination_config, schema)
+    generate.add_comments_to_configuration_map(destination_config, schema)
 
     for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration(
+        generate.add_comments_to_configuration_map(
             section_config, schema['map'][section_name], indent=generate.INDENT
         )
 

borgmatic/config/generate.py

@@ -1,8 +1,11 @@
+import io
 import os
+import re
 
 from ruamel import yaml
 
 INDENT = 4
+SEQUENCE_INDENT = 2
 
 
 def _insert_newline_before_comment(config, field_name):
@@ -15,7 +18,7 @@ def _insert_newline_before_comment(config, field_name):
     )
 
 
-def _schema_to_sample_configuration(schema, level=0):
+def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     '''
     Given a loaded configuration schema, generate and return sample config for it. Include comments
     for each section based on the schema "desc" description.
@@ -24,14 +27,29 @@ def _schema_to_sample_configuration(schema, level=0):
     if example is not None:
         return example
 
-    config = yaml.comments.CommentedMap(
-        [
-            (section_name, _schema_to_sample_configuration(section_schema, level + 1))
-            for section_name, section_schema in schema['map'].items()
-        ]
-    )
-
-    add_comments_to_configuration(config, schema, indent=(level * INDENT))
+    if 'seq' in schema:
+        config = yaml.comments.CommentedSeq(
+            [
+                _schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
+                for item_schema in schema['seq']
+            ]
+        )
+        add_comments_to_configuration_sequence(
+            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
+        )
+    elif 'map' in schema:
+        config = yaml.comments.CommentedMap(
+            [
+                (section_name, _schema_to_sample_configuration(section_schema, level + 1))
+                for section_name, section_schema in schema['map'].items()
+            ]
+        )
+        indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
+        add_comments_to_configuration_map(
+            config, schema, indent=indent, skip_first=parent_is_sequence
+        )
+    else:
+        raise ValueError('Schema at level {} is unsupported: {}'.format(level, schema))
 
     return config
 
@@ -42,13 +60,12 @@ def _comment_out_line(line):
     if not stripped_line or stripped_line.startswith('#'):
         return line
 
-    # Comment out the names of optional sections.
-    one_indent = ' ' * INDENT
-    if not line.startswith(one_indent):
-        return '# ' + line
+    # Comment out the names of optional sections, inserting the '#' after any indent for aesthetics.
+    matches = re.match(r'(\s*)', line)
+    indent_spaces = matches.group(0) if matches else ''
+    count_indent_spaces = len(indent_spaces)
 
-    # Otherwise, comment out the line, but insert the "#" after the first indent for aesthetics.
-    return '# '.join((one_indent, line[INDENT:]))
+    return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
 REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
@@ -90,7 +107,12 @@ def _render_configuration(config):
     '''
     Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
     '''
-    return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
+    dumper = yaml.YAML()
+    dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
+    rendered = io.StringIO()
+    dumper.dump(config, rendered)
+
+    return rendered.getvalue()
 
 
 def write_configuration(config_filename, rendered_config, mode=0o600):
@@ -112,13 +134,49 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
     os.chmod(config_filename, mode)
 
 
-def add_comments_to_configuration(config, schema, indent=0):
+def add_comments_to_configuration_sequence(config, schema, indent=0):
+    '''
+    If the given config sequence's items are maps, then mine the schema for the description of the
+    map's first item, and slap that atop the sequence. Indent the comment the given number of
+    characters.
+
+    Doing this for sequences of maps results in nice comments that look like:
+
+    ```
+    things:
+          # First key description. Added by this function.
+        - key: foo
+          # Second key description. Added by add_comments_to_configuration_map().
+          other: bar
+    ```
+    '''
+    if 'map' not in schema['seq'][0]:
+        return
+
+    for field_name in config[0].keys():
+        field_schema = schema['seq'][0]['map'].get(field_name, {})
+        description = field_schema.get('desc')
+
+        # No description to use? Skip it.
+        if not field_schema or not description:
+            return
+
+        config[0].yaml_set_start_comment(description, indent=indent)
+
+        # We only want the first key's description here, as the rest of the keys get commented by
+        # add_comments_to_configuration_map().
+        return
+
+
+def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
-    config before each field. This function only adds comments for the top-most config map level.
-    Indent the comment the given number of characters.
+    config mapping, before each field. Indent the comment the given number of characters.
     '''
     for index, field_name in enumerate(config.keys()):
+        if skip_first and index == 0:
+            continue
+
         field_schema = schema['map'].get(field_name, {})
         description = field_schema.get('desc')
 
@@ -127,6 +185,7 @@ def add_comments_to_configuration(config, schema, indent=0):
             continue
 
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
+
         if index > 0:
             _insert_newline_before_comment(config, field_name)
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.26'
+VERSION = '1.3.27.dev0'
 
 
 setup(

tests/integration/config/test_generate.py

@@ -40,6 +40,12 @@ def test_comment_out_line_comments_indented_option():
     assert module._comment_out_line(line) == '    # enabled: true'
 
 
+def test_comment_out_line_comments_twice_indented_option():
+    line = '        - item'
+
+    assert module._comment_out_line(line) == '        # - item'
+
+
 def test_comment_out_optional_configuration_comments_optional_config_only():
     flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
@@ -74,10 +80,10 @@ location:
     assert module._comment_out_optional_configuration(config.strip()) == expected_config.strip()
 
 
-def test_render_configuration_does_not_raise():
-    flexmock(module.yaml).should_receive('round_trip_dump')
+def test_render_configuration_converts_configuration_to_yaml_string():
+    yaml_string = module._render_configuration({'foo': 'bar'})
 
-    module._render_configuration({})
+    assert yaml_string == 'foo: bar\n'
 
 
 def test_write_configuration_does_not_raise():
@@ -107,12 +113,33 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
     module.write_configuration('config.yaml', 'config: yaml')
 
 
-def test_add_comments_to_configuration_does_not_raise():
+def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
+    config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
+    schema = {'seq': [{'type': 'str'}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
+    config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
+    schema = {'seq': [{'map': {'foo': {}}}]}
+
+    module.add_comments_to_configuration_sequence(config, schema)
+
+
+def test_add_comments_to_configuration_map_does_not_raise():
     # Ensure that it can deal with fields both in the schema and missing from the schema.
     config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
     schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
 
-    module.add_comments_to_configuration(config, schema)
+    module.add_comments_to_configuration_map(config, schema)
 
 
 def test_generate_sample_configuration_does_not_raise():

tests/unit/config/test_generate.py

@@ -1,13 +1,14 @@
 from collections import OrderedDict
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.config import generate as module
 
 
-def test_schema_to_sample_configuration_generates_config_with_examples():
+def test_schema_to_sample_configuration_generates_config_map_with_examples():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
-    flexmock(module).should_receive('add_comments_to_configuration')
+    flexmock(module).should_receive('add_comments_to_configuration_map')
     schema = {
         'map': OrderedDict(
             [
@@ -35,3 +36,38 @@ def test_schema_to_sample_configuration_generates_config_with_examples():
             ('section2', OrderedDict([('field2', 'Example 2'), ('field3', 'Example 3')])),
         ]
     )
+
+
+def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
+    flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
+    flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == ['hi']
+
+
+def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
+    flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
+    flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    schema = {
+        'seq': [
+            {
+                'map': OrderedDict(
+                    [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
+                )
+            }
+        ]
+    }
+
+    config = module._schema_to_sample_configuration(schema)
+
+    assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])]
+
+
+def test_schema_to_sample_configuration_with_unsupported_schema_raises():
+    schema = {'gobbledygook': [{'type': 'not-your'}]}
+
+    with pytest.raises(ValueError):
+        module._schema_to_sample_configuration(schema)