Merge pull request #531 from hansmi/add-exclude-from-test

Factorize and test loading of excludes from file

borg/helpers.py

@@ -235,16 +235,21 @@ def parse_timestamp(timestamp):
         return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
 
 
+def load_excludes(fh):
+    """Load and parse exclude patterns from file object. Empty lines and lines starting with '#' are ignored, but
+    whitespace is not stripped.
+    """
+    patterns = (line.rstrip('\r\n') for line in fh if not line.startswith('#'))
+    return [ExcludePattern(pattern) for pattern in patterns if pattern]
+
+
 def update_excludes(args):
-    """Merge exclude patterns from files with those on command line.
-    Empty lines and lines starting with '#' are ignored, but whitespace
-    is not stripped."""
+    """Merge exclude patterns from files with those on command line."""
     if hasattr(args, 'exclude_files') and args.exclude_files:
         if not hasattr(args, 'excludes') or args.excludes is None:
             args.excludes = []
         for file in args.exclude_files:
-            patterns = [line.rstrip('\r\n') for line in file if not line.startswith('#')]
-            args.excludes += [ExcludePattern(pattern) for pattern in patterns if pattern]
+            args.excludes += load_excludes(file)
             file.close()
 
 

borg/testsuite/helpers.py

@@ -12,7 +12,7 @@ import msgpack.fallback
 from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \
     prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
-    ProgressIndicatorPercent, ProgressIndicatorEndless
+    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes
 from . import BaseTestCase, environment_variable, FakeInputs
 
 
@@ -259,6 +259,41 @@ class OSXPatternNormalizationTestCase(BaseTestCase):
         assert e.match(str(b"ba\x80/foo", 'latin1'))
 
 
+@pytest.mark.parametrize("lines, expected", [
+    # "None" means all files, i.e. none excluded
+    ([], None),
+    (["# Comment only"], None),
+    (["*"], []),
+    (["# Comment",
+      "*/something00.txt",
+      "  whitespace\t",
+      "/whitespace/at/end of filename \t ",
+      # Whitespace before comment
+      " #/ws*",
+      # Empty line
+      "",
+      "# EOF"],
+     ["/more/data", "/home"]),
+    ])
+def test_patterns_from_file(tmpdir, lines, expected):
+    files = [
+        '/data/something00.txt', '/more/data', '/home',
+        ' #/wsfoobar',
+        '/whitespace/at/end of filename \t ',
+    ]
+
+    def evaluate(filename):
+        patterns = load_excludes(open(filename, "rt"))
+        return [path for path in files if not exclude_path(path, patterns)]
+
+    exclfile = tmpdir.join("exclude.txt")
+
+    with exclfile.open("wt") as fh:
+        fh.write("\n".join(lines))
+
+    assert evaluate(str(exclfile)) == (files if expected is None else expected)
+
+
 def test_compression_specs():
     with pytest.raises(ValueError):
         CompressionSpec('')