Merge pull request #2512 from enkore/issue/2463

Remove patterns from 1.0-maint branch

borg/archiver.py

@@ -20,9 +20,9 @@ import collections
 
 from . import __version__
 from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \
-    parse_pattern, parse_exclude_pattern, ArgparsePatternAction, ArgparsePatternFileAction, ArgparseExcludeFileAction, \
-    PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, get_cache_dir, prune_within, prune_split, \
-    Manifest, NoManifestError, remove_surrogates, format_archive, check_extension_modules, Statistics, \
+    parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, \
+    get_cache_dir, prune_within, prune_split, \
+    Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper, set_ec, \
     replace_placeholders
@@ -124,18 +124,6 @@ class Archiver:
         if self.output_list and (self.output_filter is None or status in self.output_filter):
             logger.info("%1s %s", status, remove_surrogates(path))
 
-    @staticmethod
-    def build_matcher(inclexcl_patterns, paths):
-        matcher = PatternMatcher()
-        if inclexcl_patterns:
-            matcher.add_inclexcl(inclexcl_patterns)
-        include_patterns = []
-        if paths:
-            include_patterns.extend(parse_pattern(i, PathPrefixPattern) for i in paths)
-            matcher.add(include_patterns, True)
-        matcher.fallback = not include_patterns
-        return matcher, include_patterns
-
     def do_serve(self, args):
         """Start in server mode. This command is usually not used manually.
         """
@@ -257,7 +245,8 @@ class Archiver:
     def do_create(self, args, repository, manifest=None, key=None):
         """Create new archive"""
         matcher = PatternMatcher(fallback=True)
-        matcher.add_inclexcl(args.patterns)
+        if args.excludes:
+            matcher.add(args.excludes, False)
 
         def create_inner(archive, cache):
             # Add cache dir to inode_skip list
@@ -445,7 +434,17 @@ class Archiver:
             if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )):
                 logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8')
 
-        matcher, include_patterns = self.build_matcher(args.patterns, args.paths)
+        matcher = PatternMatcher()
+        if args.excludes:
+            matcher.add(args.excludes, False)
+
+        include_patterns = []
+
+        if args.paths:
+            include_patterns.extend(parse_pattern(i, PathPrefixPattern) for i in args.paths)
+            matcher.add(include_patterns, True)
+
+        matcher.fallback = not include_patterns
 
         output_list = args.output_list
         dry_run = args.dry_run
@@ -923,9 +922,8 @@ class Archiver:
 
     helptext = collections.OrderedDict()
     helptext['patterns'] = textwrap.dedent('''
-        File patterns support four separate styles: fnmatch, shell, regular
-        expressions and path prefixes. By default, fnmatch is used for
-        `--exclude` patterns and shell-style is used for `--pattern`. If followed
+        Exclusion patterns support four separate styles, fnmatch, shell, regular
+        expressions and path prefixes. By default, fnmatch is used. If followed
         by a colon (':') the first two characters of a pattern are used as a
         style selector. Explicit style selection is necessary when a
         non-default style is desired or when the desired pattern starts with
@@ -933,12 +931,12 @@ class Archiver:
 
         `Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector `fm:`
 
-            This is the default style for --exclude and --exclude-from.
-            These patterns use a variant of shell pattern syntax, with '*' matching
-            any number of characters, '?' matching any single character, '[...]'
-            matching any single character specified, including ranges, and '[!...]'
-            matching any character not specified. For the purpose of these patterns,
-            the path separator ('\\' for Windows and '/' on other systems) is not
+            This is the default style.  These patterns use a variant of shell
+            pattern syntax, with '*' matching any number of characters, '?'
+            matching any single character, '[...]' matching any single
+            character specified, including ranges, and '[!...]' matching any
+            character not specified. For the purpose of these patterns, the
+            path separator ('\\' for Windows and '/' on other systems) is not
             treated specially. Wrap meta-characters in brackets for a literal
             match (i.e. `[?]` to match the literal character `?`). For a path
             to match a pattern, it must completely match from start to end, or
@@ -949,7 +947,6 @@ class Archiver:
 
         Shell-style patterns, selector `sh:`
 
-            This is the default style for --pattern and --patterns-from.
             Like fnmatch patterns these are similar to shell patterns. The difference
             is that the pattern may include `**/` for matching zero or more directory
             levels, `*` for matching zero or more arbitrary characters with the
@@ -1010,41 +1007,7 @@ class Archiver:
             re:^/home/[^/]\.tmp/
             sh:/home/*/.thumbnails
             EOF
-            $ borg create --exclude-from exclude.txt backup /
-
-
-        A more general and easier to use way to define filename matching patterns exists
-        with the `--pattern` and `--patterns-from` options. Using these, you may specify
-        the backup roots (starting points) and patterns for inclusion/exclusion. A
-        root path starts with the prefix `R`, followed by a path (a plain path, not a
-        file pattern). An include rule starts with the prefix +, an exclude rule starts
-        with the prefix -, both followed by a pattern.
-        Inclusion patterns are useful to include pathes that are contained in an excluded
-        path. The first matching pattern is used so if an include pattern matches before
-        an exclude pattern, the file is backed up.
-
-        Note that the default pattern style for `--pattern` and `--patterns-from` is
-        shell style (`sh:`), so those patterns behave similar to rsync include/exclude
-        patterns. The pattern style can be set via the `P` prefix.
-
-        Patterns (`--pattern`) and excludes (`--exclude`) from the command line are
-        considered first (in the order of appearance). Then patterns from `--patterns-from`
-        are added. Exclusion patterns from `--exclude-from` files are appended last.
-
-        An example `--patterns-from` file could look like that::
-
-            # "sh:" pattern style is the default, so the following line is not needed:
-            P sh
-            R /
-            # can be rebuild
-            - /home/*/.cache
-            # they're downloads for a reason
-            - /home/*/Downloads
-            # susan is a nice person
-            # include susans home
-            + /home/susan
-            # don't backup the other home directories
-            - /home/*\n\n''')
+            $ borg create --exclude-from exclude.txt backup /\n\n''')
     helptext['placeholders'] = textwrap.dedent('''
         Repository (or Archive) URLs, --prefix and --remote-path values support these
         placeholders:
@@ -1161,9 +1124,6 @@ class Archiver:
                                    help='show version number and exit')
         subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
 
-        # some empty defaults for all subparsers
-        common_parser.set_defaults(paths=[], patterns=[])
-
         serve_epilog = textwrap.dedent("""
         This command starts a repository server process. This command is usually not used manually.
         """)
@@ -1422,10 +1382,11 @@ class Archiver:
                                help='output verbose list of items (files, dirs, ...)')
         subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS',
                                help='only display items with the given status characters')
-        subparser.add_argument('-e', '--exclude', dest='patterns',
-                               type=parse_exclude_pattern, action='append',
+        subparser.add_argument('-e', '--exclude', dest='excludes',
+                               type=parse_pattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
-        subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
+        subparser.add_argument('--exclude-from', dest='exclude_files',
+                               type=argparse.FileType('r'), action='append',
                                metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
         subparser.add_argument('--exclude-caches', dest='exclude_caches',
                                action='store_true', default=False,
@@ -1436,10 +1397,6 @@ class Archiver:
         subparser.add_argument('--keep-tag-files', dest='keep_tag_files',
                                action='store_true', default=False,
                                help='keep tag files of excluded caches/directories')
-        subparser.add_argument('--pattern', action=ArgparsePatternAction,
-                               metavar="PATTERN", help='include/exclude paths matching PATTERN')
-        subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
-                               metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
         subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval',
                                type=int, default=300, metavar='SECONDS',
                                help='write checkpoint every SECONDS seconds (Default: 300)')
@@ -1486,7 +1443,7 @@ class Archiver:
         subparser.add_argument('location', metavar='ARCHIVE',
                                type=location_validator(archive=True),
                                help='name of archive to create (must be also a valid directory name)')
-        subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
+        subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                                help='paths to archive')
 
         extract_epilog = textwrap.dedent("""
@@ -1509,15 +1466,12 @@ class Archiver:
         subparser.add_argument('-n', '--dry-run', dest='dry_run',
                                default=False, action='store_true',
                                help='do not actually change any files')
-        subparser.add_argument('-e', '--exclude', dest='patterns',
-                               type=parse_exclude_pattern, action='append',
+        subparser.add_argument('-e', '--exclude', dest='excludes',
+                               type=parse_pattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
-        subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
+        subparser.add_argument('--exclude-from', dest='exclude_files',
+                               type=argparse.FileType('r'), action='append',
                                metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
-        subparser.add_argument('--pattern', action=ArgparsePatternAction,
-                               metavar="PATTERN", help='include/exclude paths matching PATTERN')
-        subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
-                               metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
         subparser.add_argument('--numeric-owner', dest='numeric_owner',
                                action='store_true', default=False,
                                help='only obey numeric user and group identifiers')
@@ -2101,11 +2055,7 @@ class Archiver:
             args = self.preprocess_args(args)
         parser = self.build_parser(args)
         args = parser.parse_args(args or ['-h'])
-        # This works around http://bugs.python.org/issue9351
-        func = getattr(args, 'func', None) or getattr(args, 'fallback_func')
-        if func == self.do_create and not args.paths:
-            # need at least 1 path but args.paths may also be populated from patterns
-            parser.error('Need at least one PATH argument.')
+        update_excludes(args)
         return args
 
     def run(self, args):

borg/helpers.py

@@ -346,57 +346,22 @@ def parse_timestamp(timestamp):
         return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
 
 
-def parse_add_pattern(patternstr, roots, patterns, fallback):
-    """Parse a pattern string and add it to roots or patterns depending on the pattern type."""
-    pattern = parse_inclexcl_pattern(patternstr, fallback=fallback)
-    if pattern.ptype is RootPath:
-        roots.append(pattern.pattern)
-    elif pattern.ptype is PatternStyle:
-        fallback = pattern.pattern
-    else:
-        patterns.append(pattern)
-    return fallback
-
-
-def load_pattern_file(fileobj, roots, patterns, fallback=None):
-    if fallback is None:
-        fallback = ShellPattern  # ShellPattern is defined later in this module
-    for patternstr in clean_lines(fileobj):
-        fallback = parse_add_pattern(patternstr, roots, patterns, fallback)
-
-
-def load_exclude_file(fileobj, patterns):
-    for patternstr in clean_lines(fileobj):
-        patterns.append(parse_exclude_pattern(patternstr))
-
-
-class ArgparsePatternAction(argparse.Action):
-    def __init__(self, nargs=1, **kw):
-        super().__init__(nargs=nargs, **kw)
-
-    def __call__(self, parser, args, values, option_string=None):
-        parse_add_pattern(values[0], args.paths, args.patterns, ShellPattern)
-
-
-class ArgparsePatternFileAction(argparse.Action):
-    def __init__(self, nargs=1, **kw):
-        super().__init__(nargs=nargs, **kw)
-
-    def __call__(self, parser, args, values, option_string=None):
-        """Load and parse patterns from a file.
-        Lines empty or starting with '#' after stripping whitespace on both line ends are ignored.
-        """
-        filename = values[0]
-        with open(filename) as f:
-            self.parse(f, args)
-
-    def parse(self, fobj, args):
-        load_pattern_file(fobj, args.paths, args.patterns)
+def load_excludes(fh):
+    """Load and parse exclude patterns from file object. Lines empty or starting with '#' after stripping whitespace on
+    both line ends are ignored.
+    """
+    patterns = (line for line in (i.strip() for i in fh) if not line.startswith('#'))
+    return [parse_pattern(pattern) for pattern in patterns if pattern]
 
 
-class ArgparseExcludeFileAction(ArgparsePatternFileAction):
-    def parse(self, fobj, args):
-        load_exclude_file(fobj, args.patterns)
+def update_excludes(args):
+    """Merge exclude patterns from files with those on command line."""
+    if hasattr(args, 'exclude_files') and args.exclude_files:
+        if not hasattr(args, 'excludes') or args.excludes is None:
+            args.excludes = []
+        for file in args.exclude_files:
+            args.excludes += load_excludes(file)
+            file.close()
 
 
 class PatternMatcher:
@@ -412,12 +377,6 @@ class PatternMatcher:
         """
         self._items.extend((i, value) for i in patterns)
 
-    def add_inclexcl(self, patterns):
-        """Add list of patterns (of type InclExclPattern) to internal list. The patterns ptype member is returned from
-        the match function when one of the given patterns matches.
-        """
-        self._items.extend(patterns)
-
     def match(self, path):
         for (pattern, value) in self._items:
             if pattern.match(path):
@@ -569,63 +528,21 @@ _PATTERN_STYLES = set([
 
 _PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES)
 
-InclExclPattern = namedtuple('InclExclPattern', 'pattern ptype')
-RootPath = object()
-PatternStyle = object()
-
-
-def get_pattern_style(prefix):
-    try:
-        return _PATTERN_STYLE_BY_PREFIX[prefix]
-    except KeyError:
-        raise ValueError("Unknown pattern style: {}".format(prefix)) from None
-
 
 def parse_pattern(pattern, fallback=FnmatchPattern):
     """Read pattern from string and return an instance of the appropriate implementation class.
     """
     if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum():
         (style, pattern) = (pattern[:2], pattern[3:])
-        cls = get_pattern_style(style)
-    else:
-        cls = fallback
-    return cls(pattern)
 
+        cls = _PATTERN_STYLE_BY_PREFIX.get(style, None)
 
-def parse_exclude_pattern(pattern, fallback=FnmatchPattern):
-    """Read pattern from string and return an instance of the appropriate implementation class.
-    """
-    epattern = parse_pattern(pattern, fallback)
-    return InclExclPattern(epattern, False)
-
-
-def parse_inclexcl_pattern(pattern, fallback=ShellPattern):
-    """Read pattern from string and return a InclExclPattern object."""
-    type_prefix_map = {
-        '-': False,
-        '+': True,
-        'R': RootPath,
-        'r': RootPath,
-        'P': PatternStyle,
-        'p': PatternStyle,
-    }
-    try:
-        ptype = type_prefix_map[pattern[0]]
-        pattern = pattern[1:].lstrip()
-        if not pattern:
-            raise ValueError("Missing pattern!")
-    except (IndexError, KeyError, ValueError):
-        raise argparse.ArgumentTypeError("Unable to parse pattern: {}".format(pattern))
-    if ptype is RootPath:
-        pobj = pattern
-    elif ptype is PatternStyle:
-        try:
-            pobj = get_pattern_style(pattern)
-        except ValueError:
-            raise argparse.ArgumentTypeError("Unable to parse pattern: {}".format(pattern))
+        if cls is None:
+            raise ValueError("Unknown pattern style: {}".format(style))
     else:
-        pobj = parse_pattern(pattern, fallback)
-    return InclExclPattern(pobj, ptype)
+        cls = fallback
+
+    return cls(pattern)
 
 
 def timestamp(s):
@@ -1462,30 +1379,6 @@ def signal_handler(sig, handler):
             signal.signal(sig, orig_handler)
 
 
-def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True):
-    """
-    clean lines (usually read from a config file):
-    1. strip whitespace (left and right), 2. remove empty lines, 3. remove comments.
-    note: only "pure comment lines" are supported, no support for "trailing comments".
-    :param lines: input line iterator (e.g. list or open text file) that gives unclean input lines
-    :param lstrip: lstrip call arguments or False, if lstripping is not desired
-    :param rstrip: rstrip call arguments or False, if rstripping is not desired
-    :param remove_comments: remove comment lines (lines starting with "#")
-    :param remove_empty: remove empty lines
-    :return: yields processed lines
-    """
-    for line in lines:
-        if lstrip is not False:
-            line = line.lstrip(lstrip)
-        if rstrip is not False:
-            line = line.rstrip(rstrip)
-        if remove_empty and not line:
-            continue
-        if remove_comments and line.startswith('#'):
-            continue
-        yield line
-
-
 def raising_signal_handler(exc_cls):
     def handler(sig_no, frame):
         # setting SIG_IGN avoids that an incoming second signal of this

borg/testsuite/archiver.py

@@ -214,7 +214,6 @@ class ArchiverTestCaseBase(BaseTestCase):
         self.keys_path = os.path.join(self.tmpdir, 'keys')
         self.cache_path = os.path.join(self.tmpdir, 'cache')
         self.exclude_file_path = os.path.join(self.tmpdir, 'excludes')
-        self.patterns_file_path = os.path.join(self.tmpdir, 'patterns')
         os.environ['BORG_KEYS_DIR'] = self.keys_path
         os.environ['BORG_CACHE_DIR'] = self.cache_path
         os.mkdir(self.input_path)
@@ -224,8 +223,6 @@ class ArchiverTestCaseBase(BaseTestCase):
         os.mkdir(self.cache_path)
         with open(self.exclude_file_path, 'wb') as fd:
             fd.write(b'input/file2\n# A comment line, then a blank line\n\n')
-        with open(self.patterns_file_path, 'wb') as fd:
-            fd.write(b'+input/file_important\n- input/file*\n# A comment line, then a blank line\n\n')
         self._old_wd = os.getcwd()
         os.chdir(self.tmpdir)
 
@@ -651,64 +648,6 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             self.cmd("extract", self.repository_location + "::test", "fm:input/file1", "fm:*file33*", "input/file2")
         self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file333"])
 
-    def test_create_without_root(self):
-        """test create without a root"""
-        self.cmd('init', self.repository_location)
-        args = ['create', self.repository_location + '::test']
-        if self.FORK_DEFAULT:
-            self.cmd(*args, exit_code=2)
-        else:
-            self.assert_raises(SystemExit, lambda: self.cmd(*args))
-
-    def test_create_pattern_root(self):
-        """test create with only a root pattern"""
-        self.cmd('init', self.repository_location)
-        self.create_regular_file('file1', size=1024 * 80)
-        self.create_regular_file('file2', size=1024 * 80)
-        output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test')
-        self.assert_in("A input/file1", output)
-        self.assert_in("A input/file2", output)
-
-    def test_create_pattern(self):
-        """test file patterns during create"""
-        self.cmd('init', self.repository_location)
-        self.create_regular_file('file1', size=1024 * 80)
-        self.create_regular_file('file2', size=1024 * 80)
-        self.create_regular_file('file_important', size=1024 * 80)
-        output = self.cmd('create', '-v', '--list',
-                          '--pattern=+input/file_important', '--pattern=-input/file*',
-                          self.repository_location + '::test', 'input')
-        self.assert_in("A input/file_important", output)
-        self.assert_not_in('file1', output)
-        self.assert_not_in('file2', output)
-
-    def test_create_pattern_file(self):
-        """test file patterns during create"""
-        self.cmd('init', self.repository_location)
-        self.create_regular_file('file1', size=1024 * 80)
-        self.create_regular_file('file2', size=1024 * 80)
-        self.create_regular_file('otherfile', size=1024 * 80)
-        self.create_regular_file('file_important', size=1024 * 80)
-        output = self.cmd('create', '-v', '--list',
-                          '--pattern=-input/otherfile', '--patterns-from=' + self.patterns_file_path,
-                          self.repository_location + '::test', 'input')
-        self.assert_in("A input/file_important", output)
-        self.assert_not_in('file1', output)
-        self.assert_not_in('file2', output)
-        self.assert_not_in('otherfile', output)
-
-    def test_extract_pattern_opt(self):
-        self.cmd('init', self.repository_location)
-        self.create_regular_file('file1', size=1024 * 80)
-        self.create_regular_file('file2', size=1024 * 80)
-        self.create_regular_file('file_important', size=1024 * 80)
-        self.cmd('create', self.repository_location + '::test', 'input')
-        with changedir('output'):
-            self.cmd('extract',
-                     '--pattern=+input/file_important', '--pattern=-input/file*',
-                     self.repository_location + '::test')
-        self.assert_equal(sorted(os.listdir('output/input')), ['file_important'])
-
     def test_exclude_caches(self):
         self.cmd('init', self.repository_location)
         self.create_regular_file('file1', size=1024 * 80)

borg/testsuite/helpers.py

@@ -9,13 +9,12 @@ import sys
 import msgpack
 import msgpack.fallback
 import time
-import argparse
 
 from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \
     prune_within, prune_split, get_cache_dir, get_keys_dir, get_security_dir, Statistics, is_slow_msgpack, \
     yes, TRUISH, FALSISH, DEFAULTISH, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
-    ProgressIndicatorPercent, ProgressIndicatorEndless, parse_pattern, load_exclude_file, load_pattern_file, \
+    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \
     PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, \
     Buffer, safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS
 
@@ -435,13 +434,8 @@ def test_invalid_unicode_pattern(pattern):
     (["pp:/"], [" #/wsfoobar", "\tstart/whitespace"]),
     (["pp:aaabbb"], None),
     (["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["/more/data", "/home"]),
-    (["/nomatch", "/more/*"],
-     ['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']),
-    # the order of exclude patterns shouldn't matter
-    (["/more/*", "/nomatch"],
-     ['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']),
     ])
-def test_exclude_patterns_from_file(tmpdir, lines, expected):
+def test_patterns_from_file(tmpdir, lines, expected):
     files = [
         '/data/something00.txt', '/more/data', '/home',
         ' #/wsfoobar',
@@ -450,10 +444,8 @@ def test_exclude_patterns_from_file(tmpdir, lines, expected):
     ]
 
     def evaluate(filename):
-        patterns = []
-        load_exclude_file(open(filename, "rt"), patterns)
         matcher = PatternMatcher(fallback=True)
-        matcher.add_inclexcl(patterns)
+        matcher.add(load_excludes(open(filename, "rt")), False)
         return [path for path in files if matcher.match(path)]
 
     exclfile = tmpdir.join("exclude.txt")
@@ -464,130 +456,6 @@ def test_exclude_patterns_from_file(tmpdir, lines, expected):
     assert evaluate(str(exclfile)) == (files if expected is None else expected)
 
 
-@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [
-    # "None" means all files, i.e. none excluded
-    ([], [], 0),
-    (["# Comment only"], [], 0),
-    (["- *"], [], 1),
-    (["+fm:*/something00.txt",
-      "-/data"], [], 2),
-    (["R /"], ["/"], 0),
-    (["R /",
-      "# comment"], ["/"], 0),
-    (["# comment",
-      "- /data",
-      "R /home"], ["/home"], 1),
-])
-def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns):
-    def evaluate(filename):
-        roots = []
-        inclexclpatterns = []
-        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
-        return roots, len(inclexclpatterns)
-    patternfile = tmpdir.join("patterns.txt")
-
-    with patternfile.open("wt") as fh:
-        fh.write("\n".join(lines))
-
-    roots, numpatterns = evaluate(str(patternfile))
-    assert roots == expected_roots
-    assert numpatterns == expected_numpatterns
-
-
-def test_switch_patterns_style():
-    patterns = """\
-        +0_initial_default_is_shell
-        p fm
-        +1_fnmatch
-        P re
-        +2_regex
-        +3_more_regex
-        P pp
-        +4_pathprefix
-        p fm
-        p sh
-        +5_shell
-    """
-    pattern_file = StringIO(patterns)
-    roots, patterns = [], []
-    load_pattern_file(pattern_file, roots, patterns)
-    assert len(patterns) == 6
-    assert isinstance(patterns[0].pattern, ShellPattern)
-    assert isinstance(patterns[1].pattern, FnmatchPattern)
-    assert isinstance(patterns[2].pattern, RegexPattern)
-    assert isinstance(patterns[3].pattern, RegexPattern)
-    assert isinstance(patterns[4].pattern, PathPrefixPattern)
-    assert isinstance(patterns[5].pattern, ShellPattern)
-
-
-@pytest.mark.parametrize("lines", [
-    (["X /data"]),  # illegal pattern type prefix
-    (["/data"]),    # need a pattern type prefix
-])
-def test_load_invalid_patterns_from_file(tmpdir, lines):
-    patternfile = tmpdir.join("patterns.txt")
-    with patternfile.open("wt") as fh:
-        fh.write("\n".join(lines))
-    filename = str(patternfile)
-    with pytest.raises(argparse.ArgumentTypeError):
-        roots = []
-        inclexclpatterns = []
-        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
-
-
-@pytest.mark.parametrize("lines, expected", [
-    # "None" means all files, i.e. none excluded
-    ([], None),
-    (["# Comment only"], None),
-    (["- *"], []),
-    # default match type is sh: for patterns -> * doesn't match a /
-    (["-*/something0?.txt"],
-     ['/data', '/data/something00.txt', '/data/subdir/something01.txt',
-      '/home', '/home/leo', '/home/leo/t', '/home/other']),
-    (["-fm:*/something00.txt"],
-     ['/data', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t', '/home/other']),
-    (["-fm:*/something0?.txt"],
-     ["/data", '/home', '/home/leo', '/home/leo/t', '/home/other']),
-    (["+/*/something0?.txt",
-      "-/data"],
-     ["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']),
-    (["+fm:*/something00.txt",
-      "-/data"],
-     ["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']),
-    # include /home/leo and exclude the rest of /home:
-    (["+/home/leo",
-      "-/home/*"],
-     ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']),
-    # wrong order, /home/leo is already excluded by -/home/*:
-    (["-/home/*",
-      "+/home/leo"],
-     ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home']),
-    (["+fm:/home/leo",
-      "-/home/"],
-     ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']),
-])
-def test_inclexcl_patterns_from_file(tmpdir, lines, expected):
-    files = [
-        '/data', '/data/something00.txt', '/data/subdir/something01.txt',
-        '/home', '/home/leo', '/home/leo/t', '/home/other'
-    ]
-
-    def evaluate(filename):
-        matcher = PatternMatcher(fallback=True)
-        roots = []
-        inclexclpatterns = []
-        load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
-        matcher.add_inclexcl(inclexclpatterns)
-        return [path for path in files if matcher.match(path)]
-
-    patternfile = tmpdir.join("patterns.txt")
-
-    with patternfile.open("wt") as fh:
-        fh.write("\n".join(lines))
-
-    assert evaluate(str(patternfile)) == (files if expected is None else expected)
-
-
 @pytest.mark.parametrize("pattern, cls", [
     ("", FnmatchPattern),