Fixing up borg module to deal with new parsed config file structures.

borgmatic/borg.py

@@ -1,10 +1,11 @@
 from datetime import datetime
+import glob
+import itertools
 import os
-import re
 import platform
+import re
 import subprocess
-from glob import glob
-from itertools import chain
+import tempfile
 
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
@@ -22,18 +23,38 @@ def initialize(storage_config, command=COMMAND):
         os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
 
 
+def _write_exclude_file(exclude_patterns=None):
+    '''
+    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
+    None if no patterns are provided.
+    '''
+    if not exclude_patterns:
+        return None
+
+    exclude_file = tempfile.NamedTemporaryFile('w')
+    exclude_file.write('\n'.join(exclude_patterns))
+    exclude_file.flush()
+
+    return exclude_file
+
+
 def create_archive(
-    excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
-    one_file_system=None, remote_path=None,
+    verbosity, storage_config, source_directories, repository, exclude_patterns=None,
+    command=COMMAND, one_file_system=None, remote_path=None,
 ):
     '''
-    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
-    list of source directories, a local or remote repository path, and a command to run, create an
-    attic archive.
+    Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
+    repository path, a list of exclude patterns, and a command to run, create an attic archive.
     '''
-    sources = re.split('\s+', source_directories)
-    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
-    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
+    sources = tuple(
+        itertools.chain.from_iterable(
+            glob.glob(directory) or [directory]
+            for directory in source_directories
+        )
+    )
+
+    exclude_file = _write_exclude_file(exclude_patterns)
+    exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)
@@ -109,12 +130,11 @@ DEFAULT_CHECKS = ('repository', 'archives')
 
 def _parse_checks(consistency_config):
     '''
-    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
-    named checks to run.
+    Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
 
     For example, given a retention config of:
 
-        {'checks': 'repository archives'}
+        {'checks': ['repository', 'archives']}
 
     This will be returned as:
 
@@ -123,14 +143,11 @@ def _parse_checks(consistency_config):
     If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
     "disabled", return an empty tuple, meaning that no checks should be run.
     '''
-    checks = consistency_config.get('checks', '').strip()
-    if not checks:
-        return DEFAULT_CHECKS
+    checks = consistency_config.get('checks', [])
+    if checks == ['disabled']:
+        return ()
 
-    return tuple(
-        check for check in consistency_config['checks'].split(' ')
-        if check.lower() not in ('disabled', '')
-    )
+    return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
 
 
 def _make_check_flags(checks, check_last=None):

borgmatic/commands/borgmatic.py

@@ -48,10 +48,7 @@ def main():  # pragma: no cover
         remote_path = config.location['remote_path']
 
         borg.initialize(config.storage)
-        # TODO: Use the new exclude_patterns.
-        borg.create_archive(
-            args.excludes_filename, args.verbosity, config.storage, **config.location
-        )
+        borg.create_archive(args.verbosity, config.storage, **config.location)
         borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
         borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:

borgmatic/tests/unit/test_borg.py

@@ -30,6 +30,20 @@ def test_initialize_without_passphrase_should_not_set_environment():
     finally:
         os.environ = orig_environ
 
+def test_write_exclude_file_does_not_raise():
+    temporary_file = flexmock(
+        name='filename',
+        write=lambda mode: None,
+        flush=lambda: None,
+    )
+    flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
+
+    module._write_exclude_file(['exclude'])
+
+
+def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
+    module._write_exclude_file([])
+
 
 def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock(STDOUT=STDOUT)
@@ -53,110 +67,100 @@ def insert_datetime_mock():
     ).mock
 
 
-CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
-CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
+CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
 
 
 def test_create_archive_should_call_borg_with_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND)
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
-        verbosity=None,
-        storage_config={},
-        source_directories='foo bar',
-        repository='repo',
-        command='borg',
-    )
-
-
-def test_create_archive_with_two_spaces_in_source_directories():
-    insert_subprocess_mock(CREATE_COMMAND)
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo  bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
-def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes():
-    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
+def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
+    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
+    insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=['exclude'],
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=VERBOSITY_SOME,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=VERBOSITY_LOTS,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={'compression': 'rle'},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
         one_file_system=True,
@@ -164,15 +168,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
 
 
 def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
         remote_path='borg1',
@@ -180,63 +185,67 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
 
 
 def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
     insert_platform_mock()
     insert_datetime_mock()
 
     module.create_archive(
-        excludes_filename='excludes',
+        exclude_patterns=None,
         verbosity=None,
         storage_config={'umask': 740},
-        source_directories='foo bar',
+        source_directories=['foo', 'bar'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_source_directories_glob_expands():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
 
 
 def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
+    flexmock(module).should_receive('_write_exclude_file')
     insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
     insert_platform_mock()
     insert_datetime_mock()
-    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+    flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        excludes_filename=None,
+        exclude_patterns=None,
         verbosity=None,
         storage_config={},
-        source_directories='foo*',
+        source_directories=['foo*'],
         repository='repo',
         command='borg',
     )
@@ -329,7 +338,7 @@ def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parame
 
 
 def test_parse_checks_returns_them_as_tuple():
-    checks = module._parse_checks({'checks': 'foo disabled bar'})
+    checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
 
     assert checks == ('foo', 'bar')
 
@@ -341,13 +350,13 @@ def test_parse_checks_with_missing_value_returns_defaults():
 
 
 def test_parse_checks_with_blank_value_returns_defaults():
-    checks = module._parse_checks({'checks': ''})
+    checks = module._parse_checks({'checks': []})
 
     assert checks == module.DEFAULT_CHECKS
 
 
 def test_parse_checks_with_disabled_returns_no_checks():
-    checks = module._parse_checks({'checks': 'disabled'})
+    checks = module._parse_checks({'checks': ['disabled']})
 
     assert checks == ()