- change argument parsing of pattern- and exclude-files:

  patterns from such files are inserted in the order of appearance on the commandline now.
- allow specifying root paths for borg create and borg extract only by root patterns ("R path")
- adopt test cases and add some test cases for pattern argument parsing

borg/archiver.py

@@ -18,9 +18,9 @@ import collections
 
 from . import __version__
 from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \
-    parse_pattern, parse_exclude_pattern, ArgparsePatternAction, PathPrefixPattern, to_localtime, timestamp, \
-    safe_timestamp, bin_to_hex, get_cache_dir, prune_within, prune_split, \
-    Manifest, NoManifestError, remove_surrogates, update_patterns, format_archive, check_extension_modules, Statistics, \
+    parse_pattern, parse_exclude_pattern, ArgparsePatternAction, ArgparsePatternFileAction, ArgparseExcludeFileAction, \
+    PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, get_cache_dir, prune_within, prune_split, \
+    Manifest, NoManifestError, remove_surrogates, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper
 from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm
@@ -999,7 +999,7 @@ class Archiver:
         patterns.
 
         Patterns (`--pattern`) and excludes (`--exclude`) from the command line are
-        considered first (in the order of appearance). Then patterns from `--pattern-from`
+        considered first (in the order of appearance). Then patterns from `--patterns-from`
         are added. Exclusion patterns from `--exclude-from` files are appended last.
 
         An example `--patterns-from` file could look like that::
@@ -1131,7 +1131,7 @@ class Archiver:
         subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
 
         # some empty defaults for all subparsers
-        common_parser.set_defaults(exclude_files=[], patterns=[], pattern_files=[])
+        common_parser.set_defaults(paths=[], patterns=[])
 
         serve_epilog = textwrap.dedent("""
         This command starts a repository server process. This command is usually not used manually.
@@ -1379,8 +1379,7 @@ class Archiver:
         subparser.add_argument('-e', '--exclude', dest='patterns',
                                type=parse_exclude_pattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
-        subparser.add_argument('--exclude-from', dest='exclude_files',
-                               type=argparse.FileType('r'), action='append',
+        subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
                                metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
         subparser.add_argument('--exclude-caches', dest='exclude_caches',
                                action='store_true', default=False,
@@ -1393,8 +1392,7 @@ class Archiver:
                                help='keep tag files of excluded caches/directories')
         subparser.add_argument('--pattern', action=ArgparsePatternAction,
                                metavar="PATTERN", help='include/exclude paths matching PATTERN')
-        subparser.add_argument('--patterns-from', dest='pattern_files',
-                               type=argparse.FileType('r'), action='append',
+        subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
                                metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
         subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval',
                                type=int, default=300, metavar='SECONDS',
@@ -1442,7 +1440,7 @@ class Archiver:
         subparser.add_argument('location', metavar='ARCHIVE',
                                type=location_validator(archive=True),
                                help='name of archive to create (must be also a valid directory name)')
-        subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
+        subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
                                help='paths to archive')
 
         extract_epilog = textwrap.dedent("""
@@ -1468,13 +1466,11 @@ class Archiver:
         subparser.add_argument('-e', '--exclude', dest='patterns',
                                type=parse_exclude_pattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
-        subparser.add_argument('--exclude-from', dest='exclude_files',
-                               type=argparse.FileType('r'), action='append',
+        subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
                                metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
         subparser.add_argument('--pattern', action=ArgparsePatternAction,
                                metavar="PATTERN", help='include/exclude paths matching PATTERN')
-        subparser.add_argument('--patterns-from', dest='pattern_files',
-                               type=argparse.FileType('r'), action='append',
+        subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
                                metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
         subparser.add_argument('--numeric-owner', dest='numeric_owner',
                                action='store_true', default=False,
@@ -2030,7 +2026,10 @@ class Archiver:
             args = self.preprocess_args(args)
         parser = self.build_parser(args)
         args = parser.parse_args(args or ['-h'])
-        update_patterns(args)
+        if args.func == self.do_create:
+            # need at least 1 path but args.paths may also be populated from patterns
+            if not args.paths:
+                parser.error('Need at least one PATH argument.')
         return args
 
     def run(self, args):

borg/helpers.py

@@ -307,40 +307,31 @@ def parse_timestamp(timestamp):
         return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
 
 
-def load_excludes(fh):
-    """Load and parse exclude patterns from file object. Lines empty or starting with '#' after stripping whitespace on
-    both line ends are ignored.
-    """
-    patterns = (line for line in (i.strip() for i in fh) if not line.startswith('#'))
-    return [parse_exclude_pattern(pattern) for pattern in patterns if pattern]
+def parse_add_pattern(patternstr, roots, patterns):
+    """Parse a pattern string and add it to roots or patterns depending on the pattern type."""
+    pattern = parse_inclexcl_pattern(patternstr)
+    if pattern.ptype is RootPath:
+        roots.append(pattern.pattern)
+    else:
+        patterns.append(pattern)
 
 
-def load_patterns(fh):
-    """Load and parse include/exclude/root patterns from file object.
-    Lines empty or starting with '#' after stripping whitespace on both line ends are ignored.
-    """
-    patternlines = (line for line in (i.strip() for i in fh) if not line.startswith('#'))
-    roots = []
-    inclexcl_patterns = []
-    for patternline in patternlines:
-        pattern = parse_inclexcl_pattern(patternline)
-        if pattern.ptype is RootPath:
-            roots.append(pattern.pattern)
-        else:
-            inclexcl_patterns.append(pattern)
-    return roots, inclexcl_patterns
+def pattern_file_iter(fileobj):
+    for line in fileobj:
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        yield line
+
 
+def load_pattern_file(fileobj, roots, patterns):
+    for patternstr in pattern_file_iter(fileobj):
+        parse_add_pattern(patternstr, roots, patterns)
 
-def update_patterns(args):
-    """Merge patterns from exclude- and pattern-files with those on command line."""
-    for file in args.pattern_files:
-        roots, inclexcl_patterns = load_patterns(file)
-        args.paths += roots
-        args.patterns += inclexcl_patterns
-        file.close()
-    for file in args.exclude_files:
-        args.patterns += load_excludes(file)
-        file.close()
+
+def load_exclude_file(fileobj, patterns):
+    for patternstr in pattern_file_iter(fileobj):
+        patterns.append(parse_exclude_pattern(patternstr))
 
 
 class ArgparsePatternAction(argparse.Action):
@@ -348,11 +339,28 @@ class ArgparsePatternAction(argparse.Action):
         super().__init__(nargs=nargs, **kw)
 
     def __call__(self, parser, args, values, option_string=None):
-        pattern = parse_inclexcl_pattern(values[0])
-        if pattern.ptype is RootPath:
-            args.paths.append(pattern.pattern)
-        else:
-            args.patterns.append(pattern)
+        parse_add_pattern(values[0], args.paths, args.patterns)
+
+
+class ArgparsePatternFileAction(argparse.Action):
+    def __init__(self, nargs=1, **kw):
+        super().__init__(nargs=nargs, **kw)
+
+    def __call__(self, parser, args, values, option_string=None):
+        """Load and parse patterns from a file.
+        Lines empty or starting with '#' after stripping whitespace on both line ends are ignored.
+        """
+        filename = values[0]
+        with open(filename) as f:
+            self.parse(f, args)
+
+    def parse(self, fobj, args):
+        load_pattern_file(fobj, args.roots, args.patterns)
+
+
+class ArgparseExcludeFileAction(ArgparsePatternFileAction):
+    def parse(self, fobj, args):
+        load_exclude_file(fobj, args.patterns)
 
 
 class PatternMatcher:

borg/testsuite/archiver.py

@@ -652,6 +652,50 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             self.cmd("extract", self.repository_location + "::test", "fm:input/file1", "fm:*file33*", "input/file2")
         self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file333"])
 
+    def test_create_without_root(self):
+        """test create without a root"""
+        self.cmd('init', self.repository_location)
+        args = [ 'create', self.repository_location + '::test' ]
+        if self.FORK_DEFAULT:
+            output = self.cmd(*args, exit_code=2)
+        else:
+            self.assert_raises(SystemExit, lambda: self.cmd(*args))
+
+    def test_create_pattern_root(self):
+        """test create with only a root pattern"""
+        self.cmd('init', self.repository_location)
+        self.create_regular_file('file1', size=1024 * 80)
+        self.create_regular_file('file2', size=1024 * 80)
+        output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test')
+        self.assert_in("A input/file1", output)
+        self.assert_in("A input/file2", output)
+
+    def test_create_pattern(self):
+        """test file patterns during create"""
+        self.cmd('init', self.repository_location)
+        self.create_regular_file('file1', size=1024 * 80)
+        self.create_regular_file('file2', size=1024 * 80)
+        self.create_regular_file('file_important', size=1024 * 80)
+        output = self.cmd('create', '-v', '--list',
+                          '--pattern=+input/file_important', '--pattern=-input/file*',
+                          self.repository_location + '::test', 'input')
+        self.assert_in("A input/file_important", output)
+        self.assert_in("A input/file_important", output)
+        self.assert_not_in('file1', output)
+        self.assert_not_in('file2', output)
+
+    def test_extract_pattern_opt(self):
+        self.cmd('init', self.repository_location)
+        self.create_regular_file('file1', size=1024 * 80)
+        self.create_regular_file('file2', size=1024 * 80)
+        self.create_regular_file('file_important', size=1024 * 80)
+        self.cmd('create', self.repository_location + '::test', 'input')
+        with changedir('output'):
+            self.cmd('extract',
+                     '--pattern=+input/file_important', '--pattern=-input/file*',
+                     self.repository_location + '::test')
+        self.assert_equal(sorted(os.listdir('output/input')), ['file_important'])
+
     def test_exclude_caches(self):
         self.cmd('init', self.repository_location)
         self.create_regular_file('file1', size=1024 * 80)

borg/testsuite/helpers.py

@@ -15,7 +15,7 @@ from ..helpers import Location, format_file_size, format_timedelta, format_line,
     prune_within, prune_split, get_cache_dir, get_keys_dir, get_security_dir, Statistics, is_slow_msgpack, \
     yes, TRUISH, FALSISH, DEFAULTISH, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
-    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, load_patterns, parse_pattern, \
+    ProgressIndicatorPercent, ProgressIndicatorEndless, parse_pattern, load_exclude_file, load_pattern_file, \
     PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, \
     Buffer
 from . import BaseTestCase, FakeInputs
@@ -434,8 +434,10 @@ def test_exclude_patterns_from_file(tmpdir, lines, expected):
     ]
 
     def evaluate(filename):
+        patterns = []
+        load_exclude_file(open(filename, "rt"), patterns)
         matcher = PatternMatcher(fallback=True)
-        matcher.add_inclexcl(load_excludes(open(filename, "rt")))
+        matcher.add_inclexcl(patterns)
         return [path for path in files if matcher.match(path)]
 
     exclfile = tmpdir.join("exclude.txt")
@@ -462,7 +464,9 @@ def test_exclude_patterns_from_file(tmpdir, lines, expected):
 ])
 def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns):
     def evaluate(filename):
-        roots, inclexclpatterns = load_patterns(open(filename, "rt"))
+        roots = []
+        inclexclpatterns = []
+        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
         return roots, len(inclexclpatterns)
     patternfile = tmpdir.join("patterns.txt")
 
@@ -484,7 +488,9 @@ def test_load_invalid_patterns_from_file(tmpdir, lines):
         fh.write("\n".join(lines))
     filename = str(patternfile)
     with pytest.raises(argparse.ArgumentTypeError):
-        roots, inclexclpatterns = load_patterns(open(filename, "rt"))
+        roots = []
+        inclexclpatterns = []
+        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
 
 
 @pytest.mark.parametrize("lines, expected", [
@@ -521,7 +527,9 @@ def test_inclexcl_patterns_from_file(tmpdir, lines, expected):
 
     def evaluate(filename):
         matcher = PatternMatcher(fallback=True)
-        roots, inclexclpatterns = load_patterns(open(filename, "rt"))
+        roots = []
+        inclexclpatterns = []
+        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
         matcher.add_inclexcl(inclexclpatterns)
         return [path for path in files if matcher.match(path)]