Improved optparse usage and implemented list archive(s) and verify_archive

dedupstore/archiver.py

@@ -29,20 +29,23 @@ class Cache(object):
         filename = os.path.join(self.path, '%s.cache' % self.store.uuid)
         if not os.path.exists(filename):
             return
-        print 'Reading cache: ', filename, '...'
+        print 'Loading cache: ', filename, '...'
         data = cPickle.loads(zlib.decompress(open(filename, 'rb').read()))
         self.chunkmap = data['chunkmap']
+        self.archives = data['archives']
         self.tid = data['tid']
         print 'done'
 
     def create(self):
         self.chunkmap = {}
+        self.archives = []
         self.tid = self.store.tid
         if self.store.tid == 0:
             return
         print 'Recreating cache...'
         for id in self.store.list(NS_ARCHIVES):
             archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, id)))
+            self.archives.append(archive['name'])
             for item in archive['items']:
                 if item['type'] == 'FILE':
                     for c in item['chunks']:
@@ -52,7 +55,7 @@ class Cache(object):
     def save(self):
         assert self.store.state == Store.OPEN
         print 'saving cache'
-        data = {'chunkmap': self.chunkmap, 'tid': self.store.tid}
+        data = {'chunkmap': self.chunkmap, 'tid': self.store.tid, 'archives': self.archives}
         filename = os.path.join(self.path, '%s.cache' % self.store.uuid)
         print 'Saving cache as:', filename
         with open(filename, 'wb') as fd:
@@ -87,13 +90,13 @@ class Cache(object):
 
 class Archiver(object):
 
-    def __init__(self):
-        self.store = Store('/tmp/store')
-        self.cache = Cache('/tmp/cache', self.store)
-
     def create_archive(self, archive_name, paths):
-#        if archive_name in self.cache.archives:
-#            raise Exception('Archive "%s" already exists' % archive_name)
+        try:
+            self.store.get(NS_ARCHIVES, archive_name)
+        except Store.DoesNotExist:
+            pass
+        else:
+            raise Exception('Archive "%s" already exists' % archive_name)
         items = []
         for path in paths:
             for root, dirs, files in os.walk(path):
@@ -106,21 +109,54 @@ class Archiver(object):
         archive = {'name': archive_name, 'items': items}
         hash = self.store.put(NS_ARCHIVES, archive_name, zlib.compress(cPickle.dumps(archive)))
         self.store.commit()
+        self.cache.archives.append(archive_name)
         self.cache.save()
 
     def delete_archive(self, archive_name):
-        archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
+        try:
+            archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
+        except Store.DoesNotExist:
+            raise Exception('Archive "%s" does not exist' % archive_name)
         self.store.delete(NS_ARCHIVES, archive_name)
-#        if not hash:
-#            raise Exception('Archive "%s" does not exist' % archive_name)
         for item in archive['items']:
             if item['type'] == 'FILE':
                 for c in item['chunks']:
                     self.cache.chunk_decref(c)
         self.store.commit()
+        self.cache.archives.remove(archive_name)
         self.cache.save()
 
+    def list_archives(self):
+        print 'Archives:'
+        for archive in sorted(self.cache.archives):
+            print archive
+
+    def list_archive(self, archive_name):
+        try:
+            archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
+        except Store.DoesNotExist:
+            raise Exception('Archive "%s" does not exist' % archive_name)
+        for item in archive['items']:
+            print item['path']
+
+    def verify_archive(self, archive_name):
+        try:
+            archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
+        except Store.DoesNotExist:
+            raise Exception('Archive "%s" does not exist' % archive_name)
+        for item in archive['items']:
+            if item['type'] == 'FILE':
+                print item['path'], '...',
+                for chunk in item['chunks']:
+                    data = self.store.get(NS_CHUNKS, chunk)
+                    if hashlib.sha1(data).digest() != chunk:
+                        print 'ERROR'
+                        break
+                else:
+                    print 'OK'
+
     def process_dir(self, path, cache):
+        path = path.lstrip('/\\:')
         print 'Directory: %s' % (path)
         return {'type': 'DIR', 'path': path}
 
@@ -134,9 +170,10 @@ class Archiver(object):
                 break
             size += len(data)
             chunks.append(cache.add_chunk(zlib.compress(data)))
+        path = path.lstrip('/\\:')
         print 'File: %s (%d chunks)' % (path, len(chunks))
         return {'type': 'FILE', 'path': path, 'size': size, 'chunks': chunks}
-    
+
     def run(self):
         parser = OptionParser()
         parser.add_option("-C", "--cache", dest="cache",
@@ -148,6 +185,7 @@ class Archiver(object):
         parser.add_option("-d", "--delete", dest="delete_archive",
                           help="delete ARCHIVE", metavar="ARCHIVE")
         parser.add_option("-l", "--list-archives", dest="list_archives",
+                        action="store_true", default=False,
                         help="list archives")
         parser.add_option("-V", "--verify", dest="verify_archive",
                         help="verify archive consistency")
@@ -156,7 +194,21 @@ class Archiver(object):
         parser.add_option("-L", "--list-archive", dest="list_archive",
                         help="verify archive consistency", metavar="ARCHIVE")
         (options, args) = parser.parse_args()
-        if options.delete_archive:
+        if options.store:
+            self.store = Store(options.store)
+        else:
+            parser.error('No store path specified')
+        if options.cache:
+            self.cache = Cache(options.cache, self.store)
+        else:
+            parser.error('No cache path specified')
+        if options.list_archives:
+            self.list_archives()
+        elif options.list_archive:
+            self.list_archive(options.list_archive)
+        elif options.verify_archive:
+            self.verify_archive(options.verify_archive)
+        elif options.delete_archive:
             self.delete_archive(options.delete_archive)
         else:
             self.create_archive(options.create_archive, args)

dedupstore/store.py

@@ -11,6 +11,12 @@ import uuid
 class Store(object):
     """
     """
+    class DoesNotExist(KeyError):
+        """"""
+
+    class AlreadyExists(KeyError):
+        """"""
+
     IDLE = 'Idle'
     OPEN = 'Open'
     ACTIVE = 'Active'
@@ -124,12 +130,12 @@ class Store(object):
             filename = os.path.join(self.path, 'txn-active', 'write', path)
             return open(filename, 'rb').read()
         if path in self.txn_delete:
-            raise Exception('Object %s does not exist' % hash.encode('hex'))
+            raise Store.DoesNotExist('Object %s:%s does not exist' % (ns.encode('hex'), id.encode('hex')))
         filename = self._filename(ns, id, os.path.join(self.path, 'data'))
         if os.path.exists(filename):
             return open(filename, 'rb').read()
         else:
-            raise Exception('Object %s does not exist' % hash.encode('hex'))
+            raise Store.DoesNotExist('Object %s:%s does not exist' % (ns.encode('hex'), id.encode('hex')))
 
     def put(self, ns, id, data):
         """
@@ -139,7 +145,7 @@ class Store(object):
         filename = self._filename(ns, id, os.path.join(self.path, 'data'))
         if (path in self.txn_write or
            (path not in self.txn_delete and os.path.exists(filename))):
-            raise Exception('Object already exists: %s:%s' % (ns.encode('hex'), id.encode('hex')))
+            raise Store.AlreadyExists('Object already exists: %s:%s' % (ns.encode('hex'), id.encode('hex')))
         if path in self.txn_delete:
             self.txn_delete.remove(path)
         if path not in self.txn_write:
@@ -164,7 +170,7 @@ class Store(object):
             if os.path.exists(filename):
                 self.txn_delete.append(path)
             else:
-                raise Exception('Object does not exist: %s' % hash.encode('hex'))
+                raise Store.DoesNotExist('Object does not exist: %s' % hash.encode('hex'))
 
     def list(self, ns, prefix='', marker=None, max_keys=1000000):
         for x in self.foo(os.path.join(self.path, 'data', ns.encode('hex')), 
@@ -209,10 +215,10 @@ class StoreTestCase(unittest.TestCase):
         self.assertEqual(self.store.tid, 0)
         self.assertEqual(self.store.state, Store.OPEN)
         self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
-        self.assertRaises(Exception, lambda: self.store.put('SOMENS', 'SOMEID', 'SOMEDATA'))
+        self.assertRaises(Store.AlreadyExists, lambda: self.store.put('SOMENS', 'SOMEID', 'SOMEDATA'))
         self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
         self.store.rollback()
-        self.assertRaises(Exception, lambda: self.store.get('SOMENS', 'SOMEID'))
+        self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
         self.assertEqual(self.store.tid, 0)
 
     def test2(self):
@@ -224,14 +230,14 @@ class StoreTestCase(unittest.TestCase):
         self.assertEqual(self.store.tid, 1)
         self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
         self.store.delete('SOMENS', 'SOMEID')
-        self.assertRaises(Exception, lambda: self.store.get('SOMENS', 'SOMEID'))
+        self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
         self.store.rollback()
         self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
         self.store.delete('SOMENS', 'SOMEID')
-        self.assertRaises(Exception, lambda: self.store.get('SOMENS', 'SOMEID'))
+        self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
         self.store.commit()
         self.assertEqual(self.store.tid, 2)
-        self.assertRaises(Exception, lambda: self.store.get('SOMENS', 'SOMEID'))
+        self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
 
     def test_list(self):
         self.store.put('SOMENS', 'SOMEID12', 'SOMEDATA')