Replace verify command with "extract --dry-run"

closes #25

CHANGES

@@ -13,6 +13,7 @@ Version 0.11
 - Fix exception during "attic create" with repeated files (#39)
 - New "--exclude-from" option for attic create/extract/verify.
 - Improved archive metadata deduplication.
+- Replace the "verify" command with "extract --dry-run" (#25)
 
 Version 0.10
 ------------

attic/archive.py

@@ -219,7 +219,13 @@ class Archive:
         cache.rollback()
         return stats
 
-    def extract_item(self, item, restore_attrs=True):
+    def extract_item(self, item, restore_attrs=True, dry_run=False):
+        if dry_run:
+            if b'chunks' in item:
+                for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
+                    pass
+            return
+
         dest = self.cwd
         if item[b'path'].startswith('/') or item[b'path'].startswith('..'):
             raise Exception('Path should be relative and local')
@@ -306,21 +312,6 @@ class Archive:
         elif not symlink:
             os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9))
 
-    def verify_file(self, item, start, result):
-        if not item[b'chunks']:
-            start(item)
-            result(item, True)
-        else:
-            start(item)
-            ids = [id for id, size, csize in item[b'chunks']]
-            try:
-                for _ in self.pipeline.fetch_many(ids, is_preloaded=True):
-                    pass
-            except Exception:
-                result(item, False)
-                return
-            result(item, True)
-
     def delete(self, cache):
         unpacker = msgpack.Unpacker(use_list=False)
         for id_, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):

attic/archiver.py

@@ -188,20 +188,25 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         patterns = adjust_patterns(args.paths, args.excludes)
         dirs = []
         for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
-            while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
-                archive.extract_item(dirs.pop(-1))
+            if not args.dry_run:
+                while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
+                    archive.extract_item(dirs.pop(-1))
             self.print_verbose(remove_surrogates(item[b'path']))
             try:
-                if stat.S_ISDIR(item[b'mode']):
-                    dirs.append(item)
-                    archive.extract_item(item, restore_attrs=False)
+                if args.dry_run:
+                    archive.extract_item(item, dry_run=True)
                 else:
-                    archive.extract_item(item)
+                    if stat.S_ISDIR(item[b'mode']):
+                        dirs.append(item)
+                        archive.extract_item(item, restore_attrs=False)
+                    else:
+                        archive.extract_item(item)
             except IOError as e:
                 self.print_error('%s: %s', remove_surrogates(item[b'path']), e)
 
-        while dirs:
-            archive.extract_item(dirs.pop(-1))
+        if not args.dry_run:
+            while dirs:
+                archive.extract_item(dirs.pop(-1))
         return self.exit_code
 
     def do_delete(self, args):
@@ -275,28 +280,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                 print('%-20s %s' % (archive.metadata[b'name'], to_localtime(archive.ts).strftime('%c')))
         return self.exit_code
 
-    def do_verify(self, args):
-        """Verify archive consistency
-        """
-        repository = self.open_repository(args.archive)
-        manifest, key = Manifest.load(repository)
-        archive = Archive(repository, key, manifest, args.archive.archive)
-        patterns = adjust_patterns(args.paths, args.excludes)
-
-        def start_cb(item):
-            self.print_verbose('%s ...', remove_surrogates(item[b'path']), newline=False)
-
-        def result_cb(item, success):
-            if success:
-                self.print_verbose('OK')
-            else:
-                self.print_verbose('ERROR')
-                self.print_error('%s: verification failed' % remove_surrogates(item[b'path']))
-        for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
-            if stat.S_ISREG(item[b'mode']) and b'chunks' in item:
-                archive.verify_file(item, start_cb, result_cb)
-        return self.exit_code
-
     def do_info(self, args):
         """Show archive details such as disk space used
         """
@@ -485,6 +468,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                           description=self.do_extract.__doc__,
                                           epilog=extract_epilog)
         subparser.set_defaults(func=self.do_extract)
+        subparser.add_argument('--dry-run', dest='dry_run',
+                               default=False, action='store_true',
+                               help='do not actually change any files')
         subparser.add_argument('-e', '--exclude', dest='excludes',
                                type=ExcludePattern, action='append',
                                metavar="PATTERN", help='exclude paths matching PATTERN')
@@ -526,24 +512,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         subparser.add_argument('-o', dest='options', type=str,
                                help='Extra mount options')
 
-        verify_epilog = '''See "attic help patterns" for more help on exclude patterns.'''
-
-        subparser = subparsers.add_parser('verify', parents=[common_parser],
-                                          description=self.do_verify.__doc__,
-                                          epilog=verify_epilog)
-        subparser.set_defaults(func=self.do_verify)
-        subparser.add_argument('-e', '--exclude', dest='excludes',
-                               type=ExcludePattern, action='append',
-                               metavar="PATTERN", help='exclude paths matching PATTERN')
-        subparser.add_argument('--exclude-from', dest='exclude_files',
-                               type=argparse.FileType('r'), action='append',
-                               metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
-        subparser.add_argument('archive', metavar='ARCHIVE',
-                               type=location_validator(archive=True),
-                               help='archive to verity integrity of')
-        subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
-                               help='paths to verify')
-
         subparser = subparsers.add_parser('info', parents=[common_parser],
                                           description=self.do_info.__doc__)
         subparser.set_defaults(func=self.do_info)

attic/helpers.py

@@ -288,9 +288,8 @@ def format_file_size(v):
         return '%d B' % v
 
 
-class IntegrityError(Exception):
-    """
-    """
+class IntegrityError(Error):
+    """Data integrity error"""
 
 
 def memoize(function):

attic/remote.py

@@ -135,7 +135,7 @@ class RemoteRepository(object):
                         elif error == b'CheckNeeded':
                             raise Repository.CheckNeeded(self.location.orig)
                         elif error == b'IntegrityError':
-                            raise IntegrityError
+                            raise IntegrityError(res)
                         raise self.RPCError(error)
                     else:
                         yield res

attic/testsuite/archiver.py

@@ -11,7 +11,7 @@ from hashlib import sha256
 from attic import xattr
 from attic.archive import Archive
 from attic.archiver import Archiver
-from attic.helpers import Manifest
+from attic.helpers import Manifest, IntegrityError
 from attic.repository import Repository
 from attic.testsuite import AtticTestCase
 from attic.crypto import bytes_to_long, num_aes_blocks
@@ -209,10 +209,10 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.attic('init', self.repository_location)
         self.attic('create', self.repository_location + '::test', 'input')
         self.attic('create', self.repository_location + '::test.2', 'input')
-        self.attic('verify', self.repository_location + '::test')
-        self.attic('verify', self.repository_location + '::test.2')
+        self.attic('extract', '--dry-run', self.repository_location + '::test')
+        self.attic('extract', '--dry-run', self.repository_location + '::test.2')
         self.attic('delete', self.repository_location + '::test')
-        self.attic('verify', self.repository_location + '::test.2')
+        self.attic('extract', '--dry-run', self.repository_location + '::test.2')
         self.attic('delete', self.repository_location + '::test.2')
         # Make sure all data except the manifest has been deleted
         repository = Repository(self.repository_path)
@@ -221,14 +221,14 @@ class ArchiverTestCase(ArchiverTestCaseBase):
     def test_corrupted_repository(self):
         self.attic('init', self.repository_location)
         self.create_src_archive('test')
-        self.attic('verify', self.repository_location + '::test')
+        self.attic('extract', '--dry-run', self.repository_location + '::test')
         self.attic('check', self.repository_location)
         name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
         fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+')
         fd.seek(100)
         fd.write('XXXX')
         fd.close()
-        self.attic('verify', self.repository_location + '::test', exit_code=1)
+        self.assert_raises(IntegrityError, lambda: self.attic('extract', '--dry-run', self.repository_location + '::test'))
         self.attic('check', self.repository_location, exit_code=1)
 
     def test_readonly_repository(self):
@@ -236,7 +236,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.create_src_archive('test')
         os.system('chmod -R ugo-w ' + self.repository_path)
         try:
-            self.attic('verify', self.repository_location + '::test')
+            self.attic('extract', '--dry-run', self.repository_location + '::test')
         finally:
             # Restore permissions so shutil.rmtree is able to delete it
             os.system('chmod -R u+w ' + self.repository_path)
@@ -369,7 +369,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         self.attic('check', self.repository_location, exit_code=1)
         self.attic('check', '--repair', self.repository_location, exit_code=0)
         self.attic('check', self.repository_location, exit_code=0)
-        self.attic('verify', self.repository_location + '::archive1', exit_code=0)
+        self.attic('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0)
 
 
 class RemoteArchiverTestCase(ArchiverTestCase):

docs/update_usage.sh

@@ -2,7 +2,7 @@
 if [ ! -d usage ]; then
   mkdir usage
 fi
-for cmd in change-passphrase check create delete extract info init list mount prune verify; do
+for cmd in change-passphrase check create delete extract info init list mount prune; do
   FILENAME="usage/$cmd.rst.inc"
   LINE=`echo -n attic $cmd | tr 'a-z- ' '-'`
   echo -e ".. _attic_$cmd:\n" > $FILENAME

docs/usage.rst

@@ -84,15 +84,6 @@ Examples
     # Extract the "src" directory but exclude object files
     $ attic extract /data/myrepo::my-files home/USERNAME/src --exclude '*.o'
 
-
-.. include:: usage/verify.rst.inc
-
-This command is similar to :ref:`attic_extract` but instead of writing any
-files to disk the command just verifies that all files are extractable and
-not corrupt. |project_name| will not compare the the archived files with the
-files on disk.
-
-
 .. include:: usage/check.rst.inc
 
 The check command verifies the consistency of a repository. Any inconsistencies