49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed includes/excludes.

NEWS

@@ -1,6 +1,8 @@
 1.1.13.dev0
  * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
    "archives", and "extract") are specified in borgmatic configuration.
+ * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
+   includes/excludes.
  * Moved issue tracker from Taiga to integrated Gitea tracker at
    https://projects.torsion.org/witten/borgmatic/issues
 
@@ -19,7 +21,7 @@
    shuts down if borgmatic is terminated (e.g. due to a system suspend).
  * #30: Support for using tilde in repository paths to reference home directory.
  * #43: Support for Borg --files-cache option for setting the files cache operation mode.
- * #45: Support for Borg --remote-ratelimit for limiting upload rate.
+ * #45: Support for Borg --remote-ratelimit option for limiting upload rate.
  * Log invoked Borg commands when at highest verbosity level.
 
 1.1.9

borgmatic/borg/create.py

@@ -31,28 +31,45 @@ def _expand_directory(directory):
     return glob.glob(expanded_directory) or [expanded_directory]
 
 
-def _write_exclude_file(exclude_patterns=None):
+def _write_pattern_file(patterns=None):
     '''
-    Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
-    None if no patterns are provided.
+    Given a sequence of patterns, write them to a named temporary file and return it. Return None
+    if no patterns are provided.
     '''
-    if not exclude_patterns:
+    if not patterns:
         return None
 
-    exclude_file = tempfile.NamedTemporaryFile('w')
-    exclude_file.write('\n'.join(exclude_patterns))
-    exclude_file.flush()
+    pattern_file = tempfile.NamedTemporaryFile('w')
+    pattern_file.write('\n'.join(patterns))
+    pattern_file.flush()
 
-    return exclude_file
+    return pattern_file
 
 
-def _make_exclude_flags(location_config, exclude_patterns_filename=None):
+def _make_pattern_flags(location_config, pattern_filename=None):
+    '''
+    Given a location config dict with a potential pattern_from option, and a filename containing any
+    additional patterns, return the corresponding Borg flags for those files as a tuple.
+    '''
+    pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
+        (pattern_filename,) if pattern_filename else ()
+    )
+
+    return tuple(
+        itertools.chain.from_iterable(
+            ('--pattern-from', pattern_filename)
+            for pattern_filename in pattern_filenames
+        )
+    )
+
+
+def _make_exclude_flags(location_config, exclude_filename=None):
     '''
     Given a location config dict with various exclude options, and a filename containing any exclude
     patterns, return the corresponding Borg flags as a tuple.
     '''
     exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
-        (exclude_patterns_filename,) if exclude_patterns_filename else ()
+        (exclude_filename,) if exclude_filename else ()
     )
     exclude_from_flags = tuple(
         itertools.chain.from_iterable(
@@ -81,10 +98,15 @@ def create_archive(
         )
     )
 
-    exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
+    pattern_file = _write_pattern_file(location_config.get('patterns'))
+    pattern_flags = _make_pattern_flags(
+        location_config,
+        pattern_file.name if pattern_file else None,
+    )
+    exclude_file = _write_pattern_file(location_config.get('exclude_patterns'))
     exclude_flags = _make_exclude_flags(
         location_config,
-        exclude_patterns_file.name if exclude_patterns_file else None,
+        exclude_file.name if exclude_file else None,
     )
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
@@ -110,7 +132,7 @@ def create_archive(
             repository=repository,
             archive_name_format=archive_name_format,
         ),
-    ) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \
+    ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
         one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
         verbosity_flags
 

borgmatic/config/schema.yaml

@@ -42,6 +42,28 @@ map:
                     repositories are backed up to in sequence.
                 example:
                     - user@backupserver:sourcehostname.borg
+            patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Any paths matching these patterns are included/excluded from backups. Globs are
+                    expanded. Note that Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details. Quoting any value if it contains leading
+                    punctuation, so it parses correctly.
+                example:
+                    - 'R /'
+                    - '- /home/*/.cache'
+                    - '+ /home/susan'
+                    - '- /home/*'
+            patterns_from:
+                seq:
+                    - type: scalar
+                desc: |
+                    Read include/exclude patterns from one or more separate named files, one pattern
+                    per line. Note that Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details.
+                example:
+                    - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
                     - type: scalar
@@ -57,7 +79,7 @@ map:
                     - type: scalar
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
-                    line.
+                    line. See the output of "borg help patterns" for more details.
                 example:
                     - /etc/borgmatic/excludes
             exclude_caches:

borgmatic/tests/unit/borg/test_create.py

@@ -58,7 +58,7 @@ def test_expand_directory_with_glob_expands():
     assert paths == ['foo', 'food']
 
 
-def test_write_exclude_file_does_not_raise():
+def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(
         name='filename',
         write=lambda mode: None,
@@ -66,11 +66,11 @@ def test_write_exclude_file_does_not_raise():
     )
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
 
-    module._write_exclude_file(['exclude'])
+    module._write_pattern_file(['exclude'])
 
 
-def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
-    module._write_exclude_file([])
+def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
+    module._write_pattern_file([])
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
@@ -78,17 +78,50 @@ def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
 
 
+def test_make_pattern_flags_includes_pattern_filename_when_given():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns': ['R /', '- /var']},
+        pattern_filename='/tmp/patterns',
+    )
+
+    assert pattern_flags == ('--pattern-from', '/tmp/patterns')
+
+
+def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': ['patterns', 'other']},
+    )
+
+    assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', 'other')
+
+
+def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': ['patterns']},
+        pattern_filename='/tmp/patterns',
+    )
+
+    assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', '/tmp/patterns')
+
+
+def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
+    pattern_flags = module._make_pattern_flags(
+        location_config={'patterns_from': None},
+    )
+
+    assert pattern_flags == ()
+
+
 def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_patterns': ['*.pyc', '/var']},
-        exclude_patterns_filename='/tmp/excludes',
+        exclude_filename='/tmp/excludes',
     )
 
     assert exclude_flags == ('--exclude-from', '/tmp/excludes')
 
 
 def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
 
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_from': ['excludes', 'other']},
@@ -98,19 +131,15 @@ def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
 
 
 def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_from': ['excludes']},
-        exclude_patterns_filename='/tmp/excludes',
+        exclude_filename='/tmp/excludes',
     )
 
     assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
 
 
 def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
-
     exclude_flags = module._make_exclude_flags(
         location_config={'exclude_from': None},
     )
@@ -154,7 +183,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
 
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND)
 
@@ -170,10 +200,31 @@ def test_create_archive_calls_borg_with_parameters():
     )
 
 
+def test_create_archive_with_patterns_calls_borg_with_patterns():
+    pattern_flags = ('--patterns-from', 'patterns')
+    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
+
+    module.create_archive(
+        verbosity=None,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'patterns': ['pattern'],
+        },
+        storage_config={},
+    )
+
+
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
+    flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
     insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
 
@@ -191,7 +242,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
 
@@ -209,7 +262,8 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
 
 def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
 
@@ -227,7 +281,8 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
 
@@ -245,7 +300,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
 
@@ -263,7 +319,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
 
@@ -282,7 +339,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
 
@@ -301,7 +359,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
 
@@ -320,7 +379,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
 
@@ -338,7 +398,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
@@ -357,7 +418,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
@@ -376,7 +438,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
 
@@ -394,7 +457,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
 
@@ -414,7 +478,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
-    flexmock(module).should_receive('_write_exclude_file').and_return(None)
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))