More cryptography work.

dedupestore/archive.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 import logging
+import msgpack
 import os
 import stat
 import sys
@@ -7,7 +8,7 @@ import sys
 from .cache import NS_ARCHIVES, NS_CHUNKS, NS_CINDEX
 from .chunkifier import chunkify
 from .crypto import CryptoManager
-from .helpers import uid2user, user2uid, gid2group, group2gid
+from .helpers import uid2user, user2uid, gid2group, group2gid, IntegrityError
 
 CHUNK_SIZE = 55001
 
@@ -26,12 +27,12 @@ class Archive(object):
 
     def load(self, id):
         self.id = id
-        archive = self.crypto.unpack_read(self.store.get(NS_ARCHIVES, self.id))
+        archive = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_ARCHIVES, self.id)))
         if archive['version'] != 1:
             raise Exception('Archive version %r not supported' % archive['version'])
         self.items = archive['items']
         self.name = archive['name']
-        cindex = self.crypto.unpack_create(self.store.get(NS_CINDEX, self.id))
+        cindex = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_CINDEX, self.id)))
         assert cindex['version'] == 1
         self.chunks = cindex['chunks']
         for i, chunk in enumerate(self.chunks):
@@ -46,15 +47,14 @@ class Archive(object):
             'ts': datetime.utcnow().isoformat(),
             'items': self.items,
         }
-        data = self.crypto.pack_read(archive)
+        data = self.crypto.encrypt_read(msgpack.packb(archive))
         self.store.put(NS_ARCHIVES, self.id, data)
         cindex = {
             'version': 1,
             'chunks': self.chunks,
         }
-        data = self.crypto.pack_create(cindex)
+        data = self.crypto.encrypt_create(msgpack.packb(cindex))
         self.store.put(NS_CINDEX, self.id, data)
-        self.crypto.store_key()
         self.store.commit()
 
     def add_chunk(self, id, size):
@@ -118,7 +118,7 @@ class Archive(object):
                     for chunk in item['chunks']:
                         id = self.chunk_idx[chunk]
                         try:
-                            fd.write(self.crypto.unpack_read(self.store.get(NS_CHUNKS, id)))
+                            fd.write(self.crypto.decrypt(self.store.get(NS_CHUNKS, id)))
                         except ValueError:
                             raise Exception('Invalid chunk checksum')
                 self.restore_stat(path, item)
@@ -146,8 +146,8 @@ class Archive(object):
                 for chunk in item['chunks']:
                     id = self.chunk_idx[chunk]
                     try:
-                        self.crypto.unpack_read(self.store.get(NS_CHUNKS, id))
-                    except ValueError:
+                        self.crypto.decrypt(self.store.get(NS_CHUNKS, id))
+                    except IntegrityError:
                         logging.error('%s ... ERROR', item['path'])
                         break
                     else:

dedupestore/cache.py

@@ -42,7 +42,7 @@ class Cache(object):
         if self.store.tid == 0:
             return
         for id in list(self.store.list(NS_CINDEX)):
-            cindex = crypto.unpack_create(self.store.get(NS_CINDEX, id))
+            cindex = msgpack.unpackb(crypto.decrypt(self.store.get(NS_CINDEX, id)))
             for id, size in cindex['chunks']:
                 try:
                     count, size = self.chunkmap[id]
@@ -65,10 +65,10 @@ class Cache(object):
         with open(self.path, 'wb') as fd:
             fd.write(data)
 
-    def add_chunk(self, id, data, crypt):
+    def add_chunk(self, id, data, crypto):
         if self.seen_chunk(id):
             return self.chunk_incref(id)
-        data = crypt.pack_read(data)
+        data = crypto.encrypt_read(data)
         csize = len(data)
         self.store.put(NS_CHUNKS, id, data)
         self.chunkmap[id] = (1, csize)

dedupestore/crypto.py

@@ -1,19 +1,18 @@
-import hashlib
-import hmac
-import msgpack
 import os
 import zlib
 
 from Crypto.Cipher import AES
+from Crypto.Hash import SHA256, HMAC
+from Crypto.Util.number import bytes_to_long, long_to_bytes
+
+from .helpers import IntegrityError
+from .oaep import OAEP
 
 
 class CryptoManager(object):
 
-    KEY_CREATE = 1
-    KEY_READ = 2
-    KEY_ID = 3
-    KEY_ARCHIVE = 4
-    KEY_CINDEX = 5
+    CREATE = '\1'
+    READ = '\2'
 
     def __init__(self, store):
         self.key_cache = {}
@@ -22,61 +21,47 @@ class CryptoManager(object):
         self.id_key = '0' * 32
         self.read_key = os.urandom(32)
         self.create_key = os.urandom(32)
-
-    def get_key(self, tid):
-        try:
-            return self.key_cache[tid]
-        except KeyError:
-            keys = self.load_key(tid)
-            self.key_cache[tid] = keys
-            return keys
-
-    def load_key(self, tid):
-        data = self.store.get('K', str(tid))
-        id = data[:32]
-        if self.id_hash(data[32:]) != id:
-            raise Exception('Invalid key object found')
-        key = msgpack.unpackb(data[32:])
-        return key['create'], key['read']
-
-    def store_key(self):
-        key = {
-            'version': 1,
-            'read': self.read_key,
-            'create': self.create_key,
-        }
-        data = msgpack.packb(key)
-        id = self.id_hash(data)
-        self.store.put('K', str(self.tid), id + data)
+        self.read_encrypted = OAEP(256, hash=SHA256).encode(self.read_key, os.urandom(32))
+        self.create_encrypted = OAEP(256, hash=SHA256).encode(self.create_key, os.urandom(32))
 
     def id_hash(self, data):
-        return hmac.new(self.id_key, data, hashlib.sha256).digest()
-
-    def pack(self, data, key):
-        data = zlib.compress(msgpack.packb(data))
-        id = hmac.new(key, data, hashlib.sha256).digest()
-        data = AES.new(key, AES.MODE_CFB, id[:16]).encrypt(data)
-        return id + msgpack.packb((1, self.tid, data))
-
-    def pack_read(self, data):
-        return self.pack(data, self.read_key)
-
-    def pack_create(self, data):
-        return self.pack(data, self.create_key)
-
-    def unpack(self, data, key_idx):
-        id = data[:32]
-        version, tid, data = msgpack.unpackb(data[32:])
-        assert version == 1
-        key = self.get_key(tid)[key_idx]
-        data = AES.new(key, AES.MODE_CFB, id[:16]).decrypt(data)
-        if hmac.new(key, data, hashlib.sha256).digest() != id:
-            raise ValueError
-        return msgpack.unpackb(zlib.decompress(data))
-
-    def unpack_read(self, data):
-        return self.unpack(data, 1)
-
-    def unpack_create(self, data):
-        return self.unpack(data, 0)
-
+        return HMAC.new(self.id_key, data, SHA256).digest()
+
+    def encrypt_read(self, data):
+        key_data = OAEP(256, hash=SHA256).encode(self.read_key, os.urandom(32))
+        #key_data = self.rsa_create.encrypt(key_data)
+        data = zlib.compress(data)
+        hash = SHA256.new(data).digest()
+        data = AES.new(self.read_key, AES.MODE_CFB, hash[:16]).encrypt(data)
+        return ''.join((self.READ, self.read_encrypted, hash, data))
+
+    def encrypt_create(self, data):
+        key_data = OAEP(256, hash=SHA256).encode(self.create_key, os.urandom(32))
+        #key_data = self.rsa_create.encrypt(key_data)
+        data = zlib.compress(data)
+        hash = SHA256.new(data).digest()
+        data = AES.new(self.create_key, AES.MODE_CFB, hash[:16]).encrypt(data)
+        return ''.join((self.CREATE, self.create_encrypted, hash, data))
+
+    def decrypt(self, data):
+        type = data[0]
+        if type == self.READ:
+            key_data = data[1:257]
+            hash = data[257:289]
+            #key_data = self.rsa_create.decrypt(key_data)
+            key = OAEP(256, hash=SHA256).decode(key_data)
+            data = AES.new(key, AES.MODE_CFB, hash[:16]).decrypt(data[289:])
+            if SHA256.new(data).digest() != hash:
+                raise IntegrityError('decryption failed')
+            return zlib.decompress(data)
+        elif type == self.CREATE:
+            key_data = data[1:257]
+            hash = data[257:289]
+            #key_data = self.rsa_create.decrypt(key_data)
+            key = OAEP(256, hash=SHA256).decode(key_data)
+            data = AES.new(key, AES.MODE_CFB, hash[:16]).decrypt(data[289:])
+            if SHA256.new(data).digest() != hash:
+                raise IntegrityError('decryption failed')
+            return zlib.decompress(data)
+        else:
+            raise Exception('Unknown pack type %d found' % ord(type))

dedupestore/helpers.py

@@ -5,6 +5,11 @@ import pwd
 import re
 
 
+class IntegrityError(Exception):
+    """
+    """
+
+
 def memoize(function):
     cache = {}
     def decorated_function(*args):