Verify cindex integrity on archive open

dedupestore/archive.py

@@ -2,6 +2,7 @@ from datetime import datetime
 import logging
 import msgpack
 import os
+import socket
 import stat
 import sys
 
@@ -26,34 +27,40 @@ class Archive(object):
 
     def load(self, id):
         self.id = id
-        archive = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_ARCHIVES, self.id)))
+        data, hash = self.crypto.decrypt(self.store.get(NS_ARCHIVES, self.id))
+        archive = msgpack.unpackb(data)
         if archive['version'] != 1:
             raise Exception('Archive version %r not supported' % archive['version'])
         self.items = archive['items']
         self.name = archive['name']
-        cindex = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_CINDEX, self.id)))
+        data, hash = self.crypto.decrypt(self.store.get(NS_CINDEX, self.id))
+        cindex = msgpack.unpackb(data)
         assert cindex['version'] == 1
+        if archive['cindex'] != hash:
+            raise Exception('decryption failed')
         self.chunks = cindex['chunks']
         for i, chunk in enumerate(self.chunks):
             self.chunk_idx[i] = chunk[0]
 
     def save(self, name):
         self.id = self.crypto.id_hash(name)
+        cindex = {
+            'version': 1,
+            'chunks': self.chunks,
+        }
+        data, cindex_hash = self.crypto.encrypt_create(msgpack.packb(cindex))
+        self.store.put(NS_CINDEX, self.id, data)
         archive = {
             'version': 1,
             'name': name,
+            'cindex': cindex_hash,
             'cmdline': sys.argv,
+            'hostname': socket.gethostname(),
             'ts': datetime.utcnow().isoformat(),
             'items': self.items,
         }
-        data = self.crypto.encrypt_read(msgpack.packb(archive))
+        data, hash = self.crypto.encrypt_read(msgpack.packb(archive))
         self.store.put(NS_ARCHIVES, self.id, data)
-        cindex = {
-            'version': 1,
-            'chunks': self.chunks,
-        }
-        data = self.crypto.encrypt_create(msgpack.packb(cindex))
-        self.store.put(NS_CINDEX, self.id, data)
         self.store.commit()
 
     def add_chunk(self, id, size):
@@ -117,7 +124,10 @@ class Archive(object):
                     for chunk in item['chunks']:
                         id = self.chunk_idx[chunk]
                         try:
-                            fd.write(self.crypto.decrypt(self.store.get(NS_CHUNKS, id)))
+                            data, hash = self.crypto.decrypt(self.store.get(NS_CHUNKS, id))
+                            if self.crypto.id_hash(data) != id:
+                                raise IntegrityError('chunk id did not match')
+                            fd.write(data)
                         except ValueError:
                             raise Exception('Invalid chunk checksum')
                 self.restore_stat(path, item)
@@ -145,7 +155,9 @@ class Archive(object):
                 for chunk in item['chunks']:
                     id = self.chunk_idx[chunk]
                     try:
-                        self.crypto.decrypt(self.store.get(NS_CHUNKS, id))
+                        data, hash = self.crypto.decrypt(self.store.get(NS_CHUNKS, id))
+                        if self.crypto.id_hash(data) != id:
+                            raise IntegrityError('chunk id did not match')
                     except IntegrityError:
                         logging.error('%s ... ERROR', item['path'])
                         break

dedupestore/cache.py

@@ -42,7 +42,8 @@ class Cache(object):
         if self.store.tid == 0:
             return
         for id in list(self.store.list(NS_CINDEX)):
-            cindex = msgpack.unpackb(crypto.decrypt(self.store.get(NS_CINDEX, id)))
+            data, hash = crypto.decrypt(self.store.get(NS_CINDEX, id))
+            cindex = msgpack.unpackb(data)
             for id, size in cindex['chunks']:
                 try:
                     count, size = self.chunkmap[id]
@@ -68,7 +69,7 @@ class Cache(object):
     def add_chunk(self, id, data, crypto):
         if self.seen_chunk(id):
             return self.chunk_incref(id)
-        data = crypto.encrypt_read(data)
+        data, hash = crypto.encrypt_read(data)
         csize = len(data)
         self.store.put(NS_CHUNKS, id, data)
         self.chunkmap[id] = (1, csize)

dedupestore/crypto.py

@@ -68,17 +68,17 @@ class CryptoManager(object):
 
     def encrypt_read(self, data):
         data = zlib.compress(data)
-        hash = SHA256.new(data).digest()
+        hash = self.id_hash(data)
         counter = Counter.new(128, initial_value=bytes_to_long(hash[:16]), allow_wraparound=True)
         data = AES.new(self.read_key, AES.MODE_CTR, '', counter=counter).encrypt(data)
-        return ''.join((self.READ, self.read_encrypted, hash, data))
+        return ''.join((self.READ, self.read_encrypted, hash, data)), hash
 
     def encrypt_create(self, data):
         data = zlib.compress(data)
-        hash = SHA256.new(data).digest()
+        hash = self.id_hash(data)
         counter = Counter.new(128, initial_value=bytes_to_long(hash[:16]), allow_wraparound=True)
         data = AES.new(self.create_key, AES.MODE_CTR, '', counter=counter).encrypt(data)
-        return ''.join((self.CREATE, self.create_encrypted, hash, data))
+        return ''.join((self.CREATE, self.create_encrypted, hash, data)), hash
 
     def decrypt_key(self, data, rsa_key):
         try:
@@ -94,16 +94,17 @@ class CryptoManager(object):
             hash = data[257:289]
             counter = Counter.new(128, initial_value=bytes_to_long(hash[:16]), allow_wraparound=True)
             data = AES.new(key, AES.MODE_CTR, counter=counter).decrypt(data[289:])
-            if SHA256.new(data).digest() != hash:
+            if self.id_hash(data) != hash:
                 raise IntegrityError('decryption failed')
-            return zlib.decompress(data)
+            return zlib.decompress(data), hash
         elif type == self.CREATE:
             key = self.decrypt_key(data[1:257], self.keychain.rsa_create)
             hash = data[257:289]
             counter = Counter.new(128, initial_value=bytes_to_long(hash[:16]), allow_wraparound=True)
             data = AES.new(key, AES.MODE_CTR, '', counter=counter).decrypt(data[289:])
-            if SHA256.new(data).digest() != hash:
+            if self.id_hash(data) != hash:
                 raise IntegrityError('decryption failed')
-            return zlib.decompress(data)
+            return zlib.decompress(data), hash
         else:
             raise Exception('Unknown pack type %d found' % ord(type))
+