Rework store design to use a single namespace

darc/__init__.py

@@ -1,4 +1,2 @@
 # This is a python package
 
-NS_CHUNK = 0
-NS_ARCHIVE_METADATA = 1

darc/archive.py

@@ -9,9 +9,8 @@ import sys
 from cStringIO import StringIO
 from xattr import xattr, XATTR_NOFOLLOW
 
-from . import NS_ARCHIVE_METADATA, NS_CHUNK
 from ._speedups import chunkify
-from .helpers import uid2user, user2uid, gid2group, group2gid, IntegrityError, \
+from .helpers import uid2user, user2uid, gid2group, group2gid, \
     Counter, encode_filename, Statistics
 
 ITEMS_BUFFER = 1024 * 1024
@@ -36,14 +35,37 @@ class Archive(object):
         self.hard_links = {}
         self.stats = Statistics()
         if name:
-            self.load(self.key.archive_hash(name))
+            manifest = Archive.read_manifest(self.store, self.key)
+            try:
+                id, ts = manifest[name]
+            except KeyError:
+                raise Archive.DoesNotExist
+            self.load(id)
+
+    @staticmethod
+    def read_manifest(store, key):
+        mid = store.meta['manifest']
+        if not mid:
+            return {}
+        mid = mid.decode('hex')
+        data = key.decrypt(mid, store.get(mid))
+        return msgpack.unpackb(data)
+
+    def write_manifest(self, manifest):
+        mid = self.store.meta['manifest']
+        if mid:
+            self.cache.chunk_decref(mid.decode('hex'))
+        if manifest:
+            data = msgpack.packb(manifest)
+            mid = self.key.id_hash(data)
+            self.cache.add_chunk(mid, data, self.stats)
+            self.store.meta['manifest'] = mid.encode('hex')
+        else:
+            self.store.meta['manifest'] = ''
 
     def load(self, id):
         self.id = id
-        try:
-            data, self.hash = self.key.decrypt(self.store.get(NS_ARCHIVE_METADATA, self.id))
-        except self.store.DoesNotExist:
-            raise self.DoesNotExist
+        data = self.key.decrypt(self.id, self.store.get(self.id))
         self.metadata = msgpack.unpackb(data)
         if self.metadata['version'] != 1:
             raise Exception('Unknown archive metadata version')
@@ -66,16 +88,15 @@ class Archive(object):
                 raise error
             assert not error
             counter.dec()
-            data, items_hash = self.key.decrypt(chunk)
-            assert self.key.id_hash(data) == id
+            data = self.key.decrypt(id, chunk)
             unpacker.feed(data)
             for item in unpacker:
                 callback(item)
-        for id, size, csize in self.metadata['items']:
+        for id in self.metadata['items']:
             # Limit the number of concurrent items requests to 10
             self.store.flush_rpc(counter, 10)
             counter.inc()
-            self.store.get(NS_CHUNK, id, callback=cb, callback_data=id)
+            self.store.get(id, callback=cb, callback_data=id)
 
     def add_item(self, item):
         self.items.write(msgpack.packb(item))
@@ -90,16 +111,15 @@ class Archive(object):
         self.items.seek(0)
         self.items.truncate()
         for chunk in chunks[:-1]:
-            self.items_ids.append(self.cache.add_chunk(self.key.id_hash(chunk),
-                                  chunk, self.stats))
+            id, _, _ = self.cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)
+            self.items_ids.append(id)
         if flush or len(chunks) == 1:
-            self.items_ids.append(self.cache.add_chunk(self.key.id_hash(chunks[-1]),
-                                  chunks[-1], self.stats))
+            id, _, _ = self.cache.add_chunk(self.key.id_hash(chunks[-1]), chunks[-1], self.stats)
+            self.items_ids.append(id)
         else:
             self.items.write(chunks[-1])
 
     def save(self, name, cache):
-        self.id = self.key.archive_hash(name)
         self.flush_items(flush=True)
         metadata = {
             'version': 1,
@@ -110,8 +130,13 @@ class Archive(object):
             'username': getuser(),
             'time': datetime.utcnow().isoformat(),
         }
-        data, self.hash = self.key.encrypt(msgpack.packb(metadata))
-        self.store.put(NS_ARCHIVE_METADATA, self.id, data)
+        data = msgpack.packb(metadata)
+        self.id = self.key.id_hash(data)
+        cache.add_chunk(self.id, data, self.stats)
+        manifest = Archive.read_manifest(self.store, self.key)
+        assert not name in manifest
+        manifest[name] = self.id, metadata['time']
+        self.write_manifest(manifest)
         self.store.commit()
         cache.commit()
 
@@ -120,8 +145,7 @@ class Archive(object):
         # the stats. The cache transaction must be rolled back afterwards
         def cb(chunk, error, id):
             assert not error
-            data, items_hash = self.key.decrypt(chunk)
-            assert self.key.id_hash(data) == id
+            data = self.key.decrypt(id, chunk)
             unpacker.feed(data)
             for item in unpacker:
                 try:
@@ -135,10 +159,11 @@ class Archive(object):
         unpacker = msgpack.Unpacker()
         cache.begin_txn()
         stats = Statistics()
-        for id, size, csize in self.metadata['items']:
-            stats.update(size, csize, self.cache.seen_chunk(id) == 1)
-            self.store.get(NS_CHUNK, id, callback=cb, callback_data=id)
-            self.cache.chunk_decref(id)
+        for id in self.metadata['items']:
+            self.store.get(id, callback=cb, callback_data=id)
+            count, size, csize = self.cache.chunks[id]
+            stats.update(size, csize, count==1)
+            self.cache.chunks[id] = count - 1, size, csize
         self.store.flush_rpc()
         cache.rollback()
         return stats
@@ -182,9 +207,7 @@ class Archive(object):
                         state['fd'] = open(path, 'wb')
                         start_cb(item)
                     assert not error
-                    data, hash = self.key.decrypt(chunk)
-                    if self.key.id_hash(data) != id:
-                        raise IntegrityError('chunk hash did not match')
+                    data = self.key.decrypt(id, chunk)
                     state['fd'].write(data)
                     if i == n - 1:
                         state['fd'].close()
@@ -198,7 +221,7 @@ class Archive(object):
                     self.restore_attrs(path, item)
                 else:
                     for i, (id, size, csize) in enumerate(item['chunks']):
-                        self.store.get(NS_CHUNK, id, callback=extract_cb, callback_data=(id, i))
+                        self.store.get(id, callback=extract_cb, callback_data=(id, i))
 
         else:
             raise Exception('Unknown archive item type %r' % item['mode'])
@@ -232,10 +255,8 @@ class Archive(object):
                 raise error
             if i == 0:
                 start(item)
-            data, hash = self.key.decrypt(chunk)
-            if self.key.id_hash(data) != id:
-                result(item, False)
-            elif i == n - 1:
+            data = self.key.decrypt(id, chunk)
+            if i == n - 1:
                 result(item, True)
         n = len(item['chunks'])
         if n == 0:
@@ -243,14 +264,12 @@ class Archive(object):
             result(item, True)
         else:
             for i, (id, size, csize) in enumerate(item['chunks']):
-                self.store.get(NS_CHUNK, id, callback=verify_chunk, callback_data=(id, i))
+                self.store.get(id, callback=verify_chunk, callback_data=(id, i))
 
     def delete(self, cache):
         def callback(chunk, error, id):
             assert not error
-            data, items_hash = self.key.decrypt(chunk)
-            if self.key.id_hash(data) != id:
-                raise IntegrityError('Chunk checksum mismatch')
+            data = self.key.decrypt(id, chunk)
             unpacker.feed(data)
             for item in unpacker:
                 try:
@@ -260,10 +279,14 @@ class Archive(object):
                     pass
             self.cache.chunk_decref(id)
         unpacker = msgpack.Unpacker()
-        for id, size, csize in self.metadata['items']:
-            self.store.get(NS_CHUNK, id, callback=callback, callback_data=id)
+        for id in self.metadata['items']:
+            self.store.get(id, callback=callback, callback_data=id)
         self.store.flush_rpc()
-        self.store.delete(NS_ARCHIVE_METADATA, self.id)
+        self.cache.chunk_decref(self.id)
+        manifest = Archive.read_manifest(self.store, self.key)
+        assert self.name in manifest
+        del manifest[self.name]
+        self.write_manifest(manifest)
         self.store.commit()
         cache.commit()
 
@@ -342,7 +365,8 @@ class Archive(object):
 
     @staticmethod
     def list_archives(store, key, cache=None):
-        for id in list(store.list(NS_ARCHIVE_METADATA)):
+        manifest = Archive.read_manifest(store, key)
+        for name, (id, ts) in manifest.items():
             archive = Archive(store, key, cache=cache)
             archive.load(id)
             yield archive

darc/archiver.py

@@ -82,7 +82,7 @@ class Archiver(object):
             diff = t - t0
             print '-' * 40
             print 'Archive name: %s' % args.archive.archive
-            print 'Archive fingerprint: %s' % archive.hash.encode('hex')
+            print 'Archive fingerprint: %s' % archive.id.encode('hex')
             print 'Start time: %s' % t0.strftime('%c')
             print 'End time: %s' % t.strftime('%c')
             print 'Duration: %s' % format_timedelta(diff)
@@ -221,7 +221,7 @@ class Archiver(object):
         archive = Archive(store, key, args.archive.archive, cache=cache)
         stats = archive.calc_stats(cache)
         print 'Name:', archive.name
-        print 'Fingerprint: %s' % archive.hash.encode('hex')
+        print 'Fingerprint: %s' % archive.id.encode('hex')
         print 'Hostname:', archive.metadata['hostname']
         print 'Username:', archive.metadata['username']
         print 'Time:', to_localtime(archive.ts).strftime('%c')

darc/cache.py

@@ -5,7 +5,7 @@ import msgpack
 import os
 import shutil
 
-from . import NS_CHUNK, NS_ARCHIVE_METADATA
+from .archive import Archive
 from .helpers import error_callback, get_cache_dir
 from .hashindex import ChunkIndex
 
@@ -18,12 +18,11 @@ class Cache(object):
         self.txn_active = False
         self.store = store
         self.key = key
-        self.path = os.path.join(get_cache_dir(), self.store.id.encode('hex'))
+        self.path = os.path.join(get_cache_dir(), store.meta['id'])
         if not os.path.exists(self.path):
             self.create()
         self.open()
-        assert self.id == store.id
-        if self.tid != store.tid:
+        if self.manifest != store.meta['manifest']:
             self.sync()
             self.commit()
 
@@ -36,8 +35,8 @@ class Cache(object):
         config = RawConfigParser()
         config.add_section('cache')
         config.set('cache', 'version', '1')
-        config.set('cache', 'store_id', self.store.id.encode('hex'))
-        config.set('cache', 'tid', '0')
+        config.set('cache', 'store', self.store.meta['id'])
+        config.set('cache', 'manifest', '')
         with open(os.path.join(self.path, 'config'), 'wb') as fd:
             config.write(fd)
         ChunkIndex.create(os.path.join(self.path, 'chunks'))
@@ -54,8 +53,8 @@ class Cache(object):
         self.config.read(os.path.join(self.path, 'config'))
         if self.config.getint('cache', 'version') != 1:
             raise Exception('%s Does not look like a darc cache')
-        self.id = self.config.get('cache', 'store_id').decode('hex')
-        self.tid = self.config.getint('cache', 'tid')
+        self.id = self.config.get('cache', 'store')
+        self.manifest = self.config.get('cache', 'manifest')
         self.chunks = ChunkIndex(os.path.join(self.path, 'chunks'))
         self.files = None
 
@@ -92,7 +91,7 @@ class Cache(object):
             with open(os.path.join(self.path, 'files'), 'wb') as fd:
                 for item in self.files.iteritems():
                     msgpack.pack(item, fd)
-        self.config.set('cache', 'tid', self.store.tid)
+        self.config.set('cache', 'manifest', self.store.meta['manifest'])
         with open(os.path.join(self.path, 'config'), 'w') as fd:
             self.config.write(fd)
         self.chunks.flush()
@@ -121,8 +120,12 @@ class Cache(object):
         """
         def cb(chunk, error, id):
             assert not error
-            data, items_hash = self.key.decrypt(chunk)
-            assert self.key.id_hash(data) == id
+            data = self.key.decrypt(id, chunk)
+            try:
+                count, size, csize = self.chunks[id]
+                self.chunks[id] = count + 1, size, csize
+            except KeyError:
+                self.chunks[id] = 1, len(data), len(chunk)
             unpacker.feed(data)
             for item in unpacker:
                 try:
@@ -138,18 +141,24 @@ class Cache(object):
         self.begin_txn()
         print 'Initializing cache...'
         self.chunks.clear()
+        # Add manifest chunk to chunk index
+        mid = self.store.meta['manifest'].decode('hex')
+        cdata = self.store.get(mid)
+        mdata = self.key.decrypt(mid, cdata)
+        self.chunks[mid] = 1, len(mdata), len(cdata)
         unpacker = msgpack.Unpacker()
-        for id in self.store.list(NS_ARCHIVE_METADATA):
-            data, hash = self.key.decrypt(self.store.get(NS_ARCHIVE_METADATA, id))
+        for name, (id, _) in Archive.read_manifest(self.store, self.key).items():
+            cdata = self.store.get(id)
+            data = self.key.decrypt(id, cdata)
+            try:
+                count, size, csize = self.chunks[id]
+                self.chunks[id] = count + 1, size, csize
+            except KeyError:
+                self.chunks[id] = 1, len(data), len(cdata)
             archive = msgpack.unpackb(data)
             print 'Analyzing archive:', archive['name']
-            for id, size, csize in archive['items']:
-                try:
-                    count, size, csize = self.chunks[id]
-                    self.chunks[id] = count + 1, size, csize
-                except KeyError:
-                    self.chunks[id] = 1, size, csize
-                self.store.get(NS_CHUNK, id, callback=cb, callback_data=id)
+            for id in archive['items']:
+                self.store.get(id, callback=cb, callback_data=id)
             self.store.flush_rpc()
 
     def add_chunk(self, id, data, stats):
@@ -158,9 +167,9 @@ class Cache(object):
         if self.seen_chunk(id):
             return self.chunk_incref(id, stats)
         size = len(data)
-        data, hash = self.key.encrypt(data)
+        data = self.key.encrypt(data)
         csize = len(data)
-        self.store.put(NS_CHUNK, id, data, callback=error_callback)
+        self.store.put(id, data, callback=error_callback)
         self.chunks[id] = (1, size, csize)
         stats.update(size, csize, True)
         return id, size, csize
@@ -182,7 +191,7 @@ class Cache(object):
         count, size, csize = self.chunks[id]
         if count == 1:
             del self.chunks[id]
-            self.store.delete(NS_CHUNK, id, callback=error_callback)
+            self.store.delete(id, callback=error_callback)
         else:
             self.chunks[id] = (count - 1, size, csize)
 

darc/helpers.py

@@ -8,7 +8,6 @@ import os
 import pwd
 import re
 import stat
-import struct
 import sys
 import time
 import urllib
@@ -121,44 +120,6 @@ def to_localtime(ts):
     return ts - timedelta(seconds=time.altzone)
 
 
-def read_set(path):
-    """Read set from disk (as int32s)
-    """
-    with open(path, 'rb') as fd:
-        data = fd.read()
-        return set(struct.unpack('<%di' % (len(data) / 4), data))
-
-
-def write_set(s, path):
-    """Write set to disk (as int32s)
-    """
-    with open(path, 'wb') as fd:
-        fd.write(struct.pack('<%di' % len(s), *s))
-
-
-def encode_long(v):
-    bytes = []
-    while True:
-        if v > 0x7f:
-            bytes.append(0x80 | (v % 0x80))
-            v >>= 7
-        else:
-            bytes.append(v)
-            return ''.join(chr(x) for x in bytes)
-
-
-def decode_long(bytes):
-    v = 0
-    base = 0
-    for x in bytes:
-        b = ord(x)
-        if b & 0x80:
-            v += (b & 0x7f) << base
-            base += 7
-        else:
-            return v + (b << base)
-
-
 def exclude_path(path, patterns):
     """Used by create and extract sub-commands to determine
     if an item should be processed or not
@@ -269,7 +230,7 @@ def format_file_size(v):
     elif v > 1024:
         return '%.2f kB' % (v / 1024.)
     else:
-        return str(v)
+        return '%d B' % v
 
 class IntegrityError(Exception):
     """

darc/key.py

@@ -23,7 +23,7 @@ class Key(object):
             self.open(self.find_key_file(store))
 
     def find_key_file(self, store):
-        id = store.id.encode('hex')
+        id = store.meta['id']
         keys_dir = get_keys_dir()
         for name in os.listdir(keys_dir):
             filename = os.path.join(keys_dir, name)
@@ -56,7 +56,6 @@ class Key(object):
         self.enc_key = key['enc_key']
         self.enc_hmac_key = key['enc_hmac_key']
         self.id_key = key['id_key']
-        self.archive_key = key['archive_key']
         self.chunk_seed = key['chunk_seed']
         self.counter = Counter.new(128, initial_value=bytes_to_long(os.urandom(16)), allow_wraparound=True)
 
@@ -93,7 +92,6 @@ class Key(object):
             'enc_key': self.enc_key,
             'enc_hmac_key': self.enc_hmac_key,
             'id_key': self.enc_key,
-            'archive_key': self.enc_key,
             'chunk_seed': self.chunk_seed,
         }
         data = self.encrypt_key_file(msgpack.packb(key), password)
@@ -129,15 +127,13 @@ class Key(object):
             if password != password2:
                 print 'Passwords do not match'
         key = Key()
-        key.store_id = store.id
+        key.store_id = store.meta['id'].decode('hex')
         # Chunk AES256 encryption key
         key.enc_key = get_random_bytes(32)
         # Chunk encryption HMAC key
         key.enc_hmac_key = get_random_bytes(32)
         # Chunk id HMAC key
         key.id_key = get_random_bytes(32)
-        # Archive name HMAC key
-        key.archive_key = get_random_bytes(32)
         # Chunkifier seed
         key.chunk_seed = bytes_to_long(get_random_bytes(4)) & 0x7fffffff
         key.save(path, password)
@@ -148,20 +144,15 @@ class Key(object):
         """
         return HMAC.new(self.id_key, data, SHA256).digest()
 
-    def archive_hash(self, data):
-        """Return HMAC hash using the "archive" HMAC key
-        """
-        return HMAC.new(self.archive_key, data, SHA256).digest()
-
     def encrypt(self, data):
         data = zlib.compress(data)
         nonce = long_to_bytes(self.counter.next_value(), 16)
         data = ''.join((nonce, AES.new(self.enc_key, AES.MODE_CTR, '',
                                        counter=self.counter).encrypt(data)))
         hash = HMAC.new(self.enc_hmac_key, data, SHA256).digest()
-        return ''.join(('\0', hash, data)), hash
+        return ''.join(('\0', hash, data))
 
-    def decrypt(self, data):
+    def decrypt(self, id, data):
         if data[0] != '\0':
             raise IntegrityError('Invalid encryption envelope')
         hash = data[1:33]
@@ -169,6 +160,8 @@ class Key(object):
             raise IntegrityError('Encryption envelope checksum mismatch')
         nonce = bytes_to_long(data[33:49])
         counter = Counter.new(128, initial_value=nonce, allow_wraparound=True)
-        data = AES.new(self.enc_key, AES.MODE_CTR, counter=counter).decrypt(data[49:])
-        return zlib.decompress(data), hash
+        data = zlib.decompress(AES.new(self.enc_key, AES.MODE_CTR, counter=counter).decrypt(data[49:]))
+        if HMAC.new(self.id_key, data, SHA256).digest() != id:
+            raise IntegrityError('Chunk id verification failed')
+        return data
 

darc/remote.py

@@ -65,7 +65,7 @@ class StoreServer(object):
         if path.startswith('/~'):
             path = path[1:]
         self.store = Store(os.path.expanduser(path), create)
-        return self.store.id, self.store.tid
+        return self.store.meta
 
 
 class RemoteStore(object):
@@ -110,7 +110,7 @@ class RemoteStore(object):
         self.msgid = 0
         self.recursion = 0
         self.odata = []
-        self.id, self.tid = self.cmd('open', (location.path, create))
+        self.meta = self.cmd('open', (location.path, create))
 
     def wait(self, write=True):
         with self.channel.lock:
@@ -160,33 +160,35 @@ class RemoteStore(object):
             else:
                 self.wait(self.odata)
 
-    def commit(self, *args):
-        self.cmd('commit', args)
-        self.tid += 1
+    def commit(self):
+        self.cmd('commit', (self.meta,))
 
     def rollback(self, *args):
         return self.cmd('rollback', args)
 
-    def get(self, ns, id, callback=None, callback_data=None):
+    def meta_get(self, *args):
+        return self.cmd('meta_get', args)
+
+    def meta_set(self, *args):
+        return self.cmd('meta_set', args)
+
+    def get(self, id, callback=None, callback_data=None):
         try:
-            return self.cmd('get', (ns, id), callback, callback_data)
+            return self.cmd('get', (id, ), callback, callback_data)
         except self.RPCError, e:
             if e.name == 'DoesNotExist':
                 raise self.DoesNotExist
             raise
 
-    def put(self, ns, id, data, callback=None, callback_data=None):
+    def put(self, id, data, callback=None, callback_data=None):
         try:
-            return self.cmd('put', (ns, id, data), callback, callback_data)
+            return self.cmd('put', (id, data), callback, callback_data)
         except self.RPCError, e:
             if e.name == 'AlreadyExists':
                 raise self.AlreadyExists
 
-    def delete(self, ns, id, callback=None, callback_data=None):
-        return self.cmd('delete', (ns, id), callback, callback_data)
-
-    def list(self, *args):
-        return self.cmd('list', args)
+    def delete(self, id, callback=None, callback_data=None):
+        return self.cmd('delete', (id, ), callback, callback_data)
 
     def flush_rpc(self, counter=None, backlog=0):
         counter = counter or self.notifier.enabled

darc/store.py

@@ -3,14 +3,15 @@ from ConfigParser import RawConfigParser
 import errno
 import fcntl
 import os
+import msgpack
 import shutil
 import struct
 import tempfile
 import unittest
 from zlib import crc32
 
-from .hashindex import NSIndex, BandIndex
-from .helpers import IntegrityError, read_set, write_set, deferrable
+from .hashindex import NSIndex
+from .helpers import IntegrityError, deferrable
 from .lrucache import LRUCache
 
 
@@ -21,7 +22,8 @@ class Store(object):
     dir/README
     dir/config
     dir/bands/<X / BANDS_PER_DIR>/<X>
-    dir/indexes/<NS>
+    dir/band
+    dir/index
     """
     DEFAULT_MAX_BAND_SIZE = 5 * 1024 * 1024
     DEFAULT_BANDS_PER_DIR = 10000
@@ -46,16 +48,17 @@ class Store(object):
         with open(os.path.join(path, 'README'), 'wb') as fd:
             fd.write('This is a DARC store')
         os.mkdir(os.path.join(path, 'bands'))
-        os.mkdir(os.path.join(path, 'indexes'))
         config = RawConfigParser()
         config.add_section('store')
         config.set('store', 'version', '1')
-        config.set('store', 'id', os.urandom(32).encode('hex'))
         config.set('store', 'bands_per_dir', self.DEFAULT_BANDS_PER_DIR)
         config.set('store', 'max_band_size', self.DEFAULT_MAX_BAND_SIZE)
-        config.add_section('state')
-        config.set('state', 'next_band', '0')
-        config.set('state', 'tid', '0')
+        config.set('store', 'next_band', '0')
+        config.add_section('meta')
+        config.set('meta', 'manifest', '')
+        config.set('meta', 'id', os.urandom(32).encode('hex'))
+        NSIndex.create(os.path.join(path, 'index'))
+        self.write_dict(os.path.join(path, 'band'), {})
         with open(os.path.join(path, 'config'), 'w') as fd:
             config.write(fd)
 
@@ -70,30 +73,38 @@ class Store(object):
         self.config.read(os.path.join(path, 'config'))
         if self.config.getint('store', 'version') != 1:
             raise Exception('%s Does not look like a darc store')
-        self.id = self.config.get('store', 'id').decode('hex')
-        self.tid = self.config.getint('state', 'tid')
-        next_band = self.config.getint('state', 'next_band')
+        next_band = self.config.getint('store', 'next_band')
         max_band_size = self.config.getint('store', 'max_band_size')
         bands_per_dir = self.config.getint('store', 'bands_per_dir')
+        self.meta = dict(self.config.items('meta'))
         self.io = BandIO(self.path, next_band, max_band_size, bands_per_dir)
         self.io.cleanup()
 
+    def read_dict(self, filename):
+        with open(filename, 'rb') as fd:
+            return msgpack.unpackb(fd.read())
+
+    def write_dict(self, filename, d):
+        with open(filename, 'wb') as fd:
+            fd.write(msgpack.packb(d))
+
     def delete_bands(self):
-        delete_path = os.path.join(self.path, 'indexes', 'delete')
+        delete_path = os.path.join(self.path, 'delete')
         if os.path.exists(delete_path):
-            bands = self.get_index('bands')
-            for band in read_set(delete_path):
+            bands = self.read_dict(os.path.join(self.path, 'band'))
+            for band in self.read_dict(delete_path):
                 assert bands.pop(band, 0) == 0
                 self.io.delete_band(band, missing_ok=True)
             os.unlink(delete_path)
+            self.write_dict(os.path.join(self.path, 'band'), bands)
 
     def begin_txn(self):
         txn_dir = os.path.join(self.path, 'txn.tmp')
         # Initialize transaction snapshot
         os.mkdir(txn_dir)
-        shutil.copytree(os.path.join(self.path, 'indexes'),
-                        os.path.join(txn_dir, 'indexes'))
         shutil.copy(os.path.join(self.path, 'config'), txn_dir)
+        shutil.copy(os.path.join(self.path, 'index'), txn_dir)
+        shutil.copy(os.path.join(self.path, 'band'), txn_dir)
         os.rename(os.path.join(self.path, 'txn.tmp'),
                   os.path.join(self.path, 'txn.active'))
         self.compact = set()
@@ -103,18 +114,21 @@ class Store(object):
         self.rollback()
         self.lock_fd.close()
 
-    def commit(self):
-        """Commit transaction, `tid` will be increased by 1
+    def commit(self, meta=None):
+        """Commit transaction
         """
+        meta = meta or self.meta
         self.compact_bands()
         self.io.close()
-        self.tid += 1
-        self.config.set('state', 'tid', self.tid)
-        self.config.set('state', 'next_band', self.io.band + 1)
+        self.config.set('store', 'next_band', self.io.band + 1)
+        self.config.remove_section('meta')
+        self.config.add_section('meta')
+        for k, v in meta.items():
+            self.config.set('meta', k, v)
         with open(os.path.join(self.path, 'config'), 'w') as fd:
             self.config.write(fd)
-        for i in self.indexes.values():
-            i.flush()
+        self.index.flush()
+        self.write_dict(os.path.join(self.path, 'band'), self.bands)
         # If we crash before this line, the transaction will be
         # rolled back by open()
         os.rename(os.path.join(self.path, 'txn.active'),
@@ -127,18 +141,18 @@ class Store(object):
         if not self.compact:
             return
         self.io.close_band()
-        def lookup(ns, key):
-            return key in self.get_index(ns)
-        bands = self.get_index('bands')
+        def lookup(key):
+            return key in self.index
+        bands = self.bands
         for band in self.compact:
             if bands[band] > 0:
-                for ns, key, data in self.io.iter_objects(band, lookup):
-                    new_band, offset = self.io.write(ns, key, data)
-                    self.indexes[ns][key] = new_band, offset
+                for key, data in self.io.iter_objects(band, lookup):
+                    new_band, offset = self.io.write(key, data)
+                    self.index[key] = new_band, offset
                     bands[band] -= 1
                     bands.setdefault(new_band, 0)
                     bands[new_band] += 1
-        write_set(self.compact, os.path.join(self.path, 'indexes', 'delete'))
+        self.write_dict(os.path.join(self.path, 'delete'), tuple(self.compact))
 
     def rollback(self):
         """
@@ -151,74 +165,53 @@ class Store(object):
         # Roll back active transaction
         txn_dir = os.path.join(self.path, 'txn.active')
         if os.path.exists(txn_dir):
-            shutil.rmtree(os.path.join(self.path, 'indexes'))
-            shutil.copytree(os.path.join(txn_dir, 'indexes'),
-                            os.path.join(self.path, 'indexes'))
             shutil.copy(os.path.join(txn_dir, 'config'), self.path)
+            shutil.copy(os.path.join(txn_dir, 'index'), self.path)
+            shutil.copy(os.path.join(txn_dir, 'band'), self.path)
             os.rename(txn_dir, os.path.join(self.path, 'txn.tmp'))
         # Remove partially removed transaction
         if os.path.exists(os.path.join(self.path, 'txn.tmp')):
             shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
-        self.indexes = {}
+        self.index = NSIndex(os.path.join(self.path, 'index'))
+        self.bands = self.read_dict(os.path.join(self.path, 'band'))
         self.txn_active = False
 
-    def get_index(self, ns):
-        try:
-            return self.indexes[ns]
-        except KeyError:
-            if ns == 'bands':
-                filename = os.path.join(self.path, 'indexes', 'bands')
-                cls = BandIndex
-            else:
-                filename = os.path.join(self.path, 'indexes', 'ns%d' % ns)
-                cls = NSIndex
-            if os.path.exists(filename):
-                self.indexes[ns] = cls(filename)
-            else:
-                self.indexes[ns] = cls.create(filename)
-            return self.indexes[ns]
-
     @deferrable
-    def get(self, ns, id):
+    def get(self, id):
         try:
-            band, offset = self.get_index(ns)[id]
-            return self.io.read(band, offset, ns, id)
+            band, offset = self.index[id]
+            return self.io.read(band, offset, id)
         except KeyError:
             raise self.DoesNotExist
 
     @deferrable
-    def put(self, ns, id, data):
+    def put(self, id, data):
         if not self.txn_active:
             self.begin_txn()
-        band, offset = self.io.write(ns, id, data)
-        bands = self.get_index('bands')
-        bands.setdefault(band, 0)
-        bands[band] += 1
-        self.get_index(ns)[id] = band, offset
+        band, offset = self.io.write(id, data)
+        self.bands.setdefault(band, 0)
+        self.bands[band] += 1
+        self.index[id] = band, offset
 
     @deferrable
-    def delete(self, ns, id):
+    def delete(self, id):
         if not self.txn_active:
             self.begin_txn()
         try:
-            band, offset = self.get_index(ns).pop(id)
-            self.get_index('bands')[band] -= 1
+            band, offset = self.index.pop(id)
+            self.bands[band] -= 1
             self.compact.add(band)
         except KeyError:
             raise self.DoesNotExist
 
-    @deferrable
-    def list(self, ns, marker=None, limit=1000000):
-        return [key for key, value in self.get_index(ns).iteritems(marker=marker, limit=limit)]
-
     def flush_rpc(self, *args):
         pass
 
 
 class BandIO(object):
 
-    header_fmt = struct.Struct('<IBIB32s')
-    assert header_fmt.size == 42
+    header_fmt = struct.Struct('<IBI32s')
+    assert header_fmt.size == 41
 
     def __init__(self, path, nextband, limit, bands_per_dir, capacity=100):
         self.path = path
@@ -265,12 +258,12 @@ class BandIO(object):
             if not missing_ok or e.errno != errno.ENOENT:
                 raise
 
-    def read(self, band, offset, ns, id):
+    def read(self, band, offset, id):
         fd = self.get_fd(band)
         fd.seek(offset)
         data = fd.read(self.header_fmt.size)
-        size, magic, hash, ns_, id_ = self.header_fmt.unpack(data)
-        if magic != 0 or ns != ns_ or id != id_:
+        size, magic, hash, id_ = self.header_fmt.unpack(data)
+        if magic != 0 or id != id_:
             raise IntegrityError('Invalid band entry header')
         data = fd.read(size - self.header_fmt.size)
         if crc32(data) & 0xffffffff != hash:
@@ -285,20 +278,20 @@ class BandIO(object):
         offset = 8
         data = fd.read(self.header_fmt.size)
         while data:
-            size, magic, hash, ns, key = self.header_fmt.unpack(data)
+            size, magic, hash, key = self.header_fmt.unpack(data)
             if magic != 0:
                 raise IntegrityError('Unknown band entry header')
             offset += size
-            if lookup(ns, key):
+            if lookup(key):
                 data = fd.read(size - self.header_fmt.size)
                 if crc32(data) & 0xffffffff != hash:
                     raise IntegrityError('Band checksum mismatch')
-                yield ns, key, data
+                yield key, data
             else:
                 fd.seek(offset)
             data = fd.read(self.header_fmt.size)
 
-    def write(self, ns, id, data):
+    def write(self, id, data):
         size = len(data) + self.header_fmt.size
         if self.offset and self.offset + size > self.limit:
             self.close_band()
@@ -309,7 +302,7 @@ class BandIO(object):
             self.offset = 8
         offset = self.offset
         hash = crc32(data) & 0xffffffff
-        fd.write(self.header_fmt.pack(size, 0, hash, ns, id))
+        fd.write(self.header_fmt.pack(size, 0, hash, id))
         fd.write(data)
         self.offset += size
         return self.band, offset
@@ -329,37 +322,15 @@ class StoreTestCase(unittest.TestCase):
         shutil.rmtree(self.tmppath)
 
     def test1(self):
-        self.assertEqual(self.store.tid, 0)
         for x in range(100):
-            self.store.put(0, '%-32d' % x, 'SOMEDATA')
+            self.store.put('%-32d' % x, 'SOMEDATA')
         key50 = '%-32d' % 50
-        self.assertEqual(self.store.get(0, key50), 'SOMEDATA')
-        self.store.delete(0, key50)
-        self.assertRaises(self.store.DoesNotExist, lambda: self.store.get(0, key50))
+        self.assertEqual(self.store.get(key50), 'SOMEDATA')
+        self.store.delete(key50)
+        self.assertRaises(self.store.DoesNotExist, lambda: self.store.get(key50))
         self.store.commit()
-        self.assertEqual(self.store.tid, 1)
         self.store.close()
         store2 = Store(os.path.join(self.tmppath, 'store'))
-        self.assertEqual(store2.tid, 1)
-        keys = list(store2.list(0))
-        for x in range(50):
-            key = '%-32d' % x
-            self.assertEqual(store2.get(0, key), 'SOMEDATA')
-        self.assertRaises(store2.DoesNotExist, lambda: store2.get(0, key50))
-        assert key50 not in keys
-        for x in range(51, 100):
-            key = '%-32d' % x
-            assert key in keys
-            self.assertEqual(store2.get(0, key), 'SOMEDATA')
-        self.assertEqual(len(keys), 99)
-        for x in range(50):
-            key = '%-32d' % x
-            store2.delete(0, key)
-        self.assertEqual(len(list(store2.list(0))), 49)
-        for x in range(51, 100):
-            key = '%-32d' % x
-            store2.delete(0, key)
-        self.assertEqual(len(list(store2.list(0))), 0)
 
 
 def suite():