Merge pull request #2010 from costela/master

change dir_is_tagged to use os.path.exists()

docs/changes.rst

@@ -126,6 +126,18 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
+Version 1.1.0b4 (not released yet)
+----------------------------------
+
+New features:
+
+- the --exclude-if-present option now supports tagging a folder with any
+  filesystem object type (file, folder, etc), instead of expecting only files
+  as tags, #1999
+- the --keep-tag-files option has been deprecated in favor of the new 
+  --keep-exclude-tags, to account for the change mentioned above.
+
+
 Version 1.1.0b3 (2017-01-15)
 ----------------------------
 

src/borg/archive.py

@@ -1439,7 +1439,7 @@ class ArchiveRecreater:
         return archive_name.endswith('.recreate')
 
     def __init__(self, repository, manifest, key, cache, matcher,
-                 exclude_caches=False, exclude_if_present=None, keep_tag_files=False,
+                 exclude_caches=False, exclude_if_present=None, keep_exclude_tags=False,
                  chunker_params=None, compression=None, compression_files=None, always_recompress=False,
                  dry_run=False, stats=False, progress=False, file_status_printer=None,
                  checkpoint_interval=1800):
@@ -1451,7 +1451,7 @@ class ArchiveRecreater:
         self.matcher = matcher
         self.exclude_caches = exclude_caches
         self.exclude_if_present = exclude_if_present or []
-        self.keep_tag_files = keep_tag_files
+        self.keep_exclude_tags = keep_exclude_tags
 
         self.rechunkify = chunker_params is not None
         if self.rechunkify:
@@ -1591,7 +1591,7 @@ class ArchiveRecreater:
     def matcher_add_tagged_dirs(self, archive):
         """Add excludes to the matcher created by exclude_cache and exclude_if_present."""
         def exclude(dir, tag_item):
-            if self.keep_tag_files:
+            if self.keep_exclude_tags:
                 tag_files.append(PathPrefixPattern(tag_item.path))
                 tagged_dirs.append(FnmatchPattern(dir + '/'))
             else:
@@ -1607,10 +1607,10 @@ class ArchiveRecreater:
                 filter=lambda item: item.path.endswith(CACHE_TAG_NAME) or matcher.match(item.path)):
             if item.path.endswith(CACHE_TAG_NAME):
                 cachedir_masters[item.path] = item
+            dir, tag_file = os.path.split(item.path)
+            if tag_file in self.exclude_if_present:
+                exclude(dir, item)
             if stat.S_ISREG(item.mode):
-                dir, tag_file = os.path.split(item.path)
-                if tag_file in self.exclude_if_present:
-                    exclude(dir, item)
                 if self.exclude_caches and tag_file == CACHE_TAG_NAME:
                     if 'chunks' in item:
                         file = open_item(archive, item)

src/borg/archiver.py

@@ -346,7 +346,7 @@ class Archiver:
                 else:
                     restrict_dev = None
                 self._process(archive, cache, matcher, args.exclude_caches, args.exclude_if_present,
-                              args.keep_tag_files, skip_inodes, path, restrict_dev,
+                              args.keep_exclude_tags, skip_inodes, path, restrict_dev,
                               read_special=args.read_special, dry_run=dry_run, st=st)
             if not dry_run:
                 archive.save(comment=args.comment, timestamp=args.timestamp)
@@ -382,7 +382,7 @@ class Archiver:
         return self.exit_code
 
     def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present,
-                 keep_tag_files, skip_inodes, path, restrict_dev,
+                 keep_exclude_tags, skip_inodes, path, restrict_dev,
                  read_special=False, dry_run=False, st=None):
         if not matcher.match(path):
             self.print_file_status('x', path)
@@ -419,11 +419,11 @@ class Archiver:
             if recurse:
                 tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present)
                 if tag_paths:
-                    if keep_tag_files and not dry_run:
+                    if keep_exclude_tags and not dry_run:
                         archive.process_dir(path, st)
                         for tag_path in tag_paths:
                             self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
-                                          keep_tag_files, skip_inodes, tag_path, restrict_dev,
+                                          keep_exclude_tags, skip_inodes, tag_path, restrict_dev,
                                           read_special=read_special, dry_run=dry_run)
                     return
             if not dry_run:
@@ -438,7 +438,7 @@ class Archiver:
                     for dirent in entries:
                         normpath = os.path.normpath(dirent.path)
                         self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
-                                      keep_tag_files, skip_inodes, normpath, restrict_dev,
+                                      keep_exclude_tags, skip_inodes, normpath, restrict_dev,
                                       read_special=read_special, dry_run=dry_run)
         elif stat.S_ISLNK(st.st_mode):
             if not dry_run:
@@ -1151,7 +1151,7 @@ class Archiver:
 
         recreater = ArchiveRecreater(repository, manifest, key, cache, matcher,
                                      exclude_caches=args.exclude_caches, exclude_if_present=args.exclude_if_present,
-                                     keep_tag_files=args.keep_tag_files, chunker_params=args.chunker_params,
+                                     keep_exclude_tags=args.keep_exclude_tags, chunker_params=args.chunker_params,
                                      compression=args.compression, compression_files=args.compression_files,
                                      always_recompress=args.always_recompress,
                                      progress=args.progress, stats=args.stats,
@@ -1571,6 +1571,7 @@ class Archiver:
         deprecations = [
             # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),
             ('--list-format', '--format', 'Warning: "--list-format" has been deprecated. Use "--format" instead.'),
+            ('--keep-tag-files', '--keep-exclude-tags', 'Warning: "--keep-tag-files" has been deprecated. Use "--keep-exclude-tags" instead.'),
         ]
         for i, arg in enumerate(args[:]):
             for old_name, new_name, warning in deprecations:
@@ -1974,11 +1975,13 @@ class Archiver:
                                    help='exclude directories that contain a CACHEDIR.TAG file ('
                                         'http://www.brynosaurus.com/cachedir/spec.html)')
         exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
-                                   metavar='FILENAME', action='append', type=str,
-                                   help='exclude directories that contain the specified file')
-        exclude_group.add_argument('--keep-tag-files', dest='keep_tag_files',
+                                   metavar='NAME', action='append', type=str,
+                                   help='exclude directories that are tagged by containing a filesystem object with \
+                                         the given NAME')
+        exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
                                    action='store_true', default=False,
-                                   help='keep tag files of excluded caches/directories')
+                                   help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \
+                                         excluded caches/directories')
 
         fs_group = subparser.add_argument_group('Filesystem options')
         fs_group.add_argument('-x', '--one-file-system', dest='one_file_system',
@@ -2562,11 +2565,13 @@ class Archiver:
                                    help='exclude directories that contain a CACHEDIR.TAG file ('
                                         'http://www.brynosaurus.com/cachedir/spec.html)')
         exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
-                                   metavar='FILENAME', action='append', type=str,
-                                   help='exclude directories that contain the specified file')
-        exclude_group.add_argument('--keep-tag-files', dest='keep_tag_files',
+                                   metavar='NAME', action='append', type=str,
+                                   help='exclude directories that are tagged by containing a filesystem object with \
+                                         the given NAME')
+        exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
                                    action='store_true', default=False,
-                                   help='keep tag files of excluded caches/directories')
+                                   help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \
+                                         excluded caches/directories')
 
         archive_group = subparser.add_argument_group('Archive options')
         archive_group.add_argument('--target', dest='target', metavar='TARGET', default=None,

src/borg/helpers.py

@@ -633,9 +633,9 @@ def dir_is_cachedir(path):
 
 def dir_is_tagged(path, exclude_caches, exclude_if_present):
     """Determines whether the specified path is excluded by being a cache
-    directory or containing user-specified tag files. Returns a list of the
-    paths of the tag files (either CACHEDIR.TAG or the matching
-    user-specified files).
+    directory or containing user-specified tag files/directories. Returns a
+    list of the paths of the tag files/directories (either CACHEDIR.TAG or the
+    matching user-specified files/directories).
     """
     tag_paths = []
     if exclude_caches and dir_is_cachedir(path):
@@ -643,7 +643,7 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
     if exclude_if_present is not None:
         for tag in exclude_if_present:
             tag_path = os.path.join(path, tag)
-            if os.path.isfile(tag_path):
+            if os.path.exists(tag_path):
                 tag_paths.append(tag_path)
     return tag_paths
 

src/borg/testsuite/archiver.py

@@ -898,12 +898,12 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.create_regular_file('file1', size=1024 * 80)
         self.create_regular_file('tagged1/.NOBACKUP')
         self.create_regular_file('tagged2/00-NOBACKUP')
-        self.create_regular_file('tagged3/.NOBACKUP/file2')
+        self.create_regular_file('tagged3/.NOBACKUP/file2', size=1024)
 
     def _assert_test_tagged(self):
         with changedir('output'):
             self.cmd('extract', self.repository_location + '::test')
-        self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged3'])
+        self.assert_equal(sorted(os.listdir('output/input')), ['file1'])
 
     def test_exclude_tagged(self):
         self._create_test_tagged()
@@ -922,13 +922,13 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.create_regular_file('file0', size=1024)
         self.create_regular_file('tagged1/.NOBACKUP1')
         self.create_regular_file('tagged1/file1', size=1024)
-        self.create_regular_file('tagged2/.NOBACKUP2')
+        self.create_regular_file('tagged2/.NOBACKUP2/subfile1', size=1024)
         self.create_regular_file('tagged2/file2', size=1024)
         self.create_regular_file('tagged3/%s' % CACHE_TAG_NAME,
                                  contents=CACHE_TAG_CONTENTS + b' extra stuff')
         self.create_regular_file('tagged3/file3', size=1024)
         self.create_regular_file('taggedall/.NOBACKUP1')
-        self.create_regular_file('taggedall/.NOBACKUP2')
+        self.create_regular_file('taggedall/.NOBACKUP2/subfile1', size=1024)
         self.create_regular_file('taggedall/%s' % CACHE_TAG_NAME,
                                  contents=CACHE_TAG_CONTENTS + b' extra stuff')
         self.create_regular_file('taggedall/file4', size=1024)
@@ -943,17 +943,22 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.assert_equal(sorted(os.listdir('output/input/taggedall')),
                           ['.NOBACKUP1', '.NOBACKUP2', CACHE_TAG_NAME, ])
 
+    def test_exclude_keep_tagged_deprecation(self):
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        output_warn = self.cmd('create', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', src_dir)
+        self.assert_in('--keep-tag-files" has been deprecated.', output_warn)
+
     def test_exclude_keep_tagged(self):
         self._create_test_keep_tagged()
         self.cmd('create', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2',
-                 '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input')
+                 '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test', 'input')
         self._assert_test_keep_tagged()
 
     def test_recreate_exclude_keep_tagged(self):
         self._create_test_keep_tagged()
         self.cmd('create', self.repository_location + '::test', 'input')
         self.cmd('recreate', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2',
-                 '--exclude-caches', '--keep-tag-files', self.repository_location + '::test')
+                 '--exclude-caches', '--keep-exclude-tags', self.repository_location + '::test')
         self._assert_test_keep_tagged()
 
     @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='Linux capabilities test, requires fakeroot >= 1.20.2')