Merge branch 'merge' into merge-all

CHANGES

@@ -3,6 +3,14 @@ Attic Changelog
 
 Here you can see the full list of changes between each Attic release.
 
+Version 0.16
+------------
+
+(bugfix release, released on X)
+- Fix "All archives" output for attic info. (#183)
+- More user friendly error message when repository key file is not found (#236)
+- Fix parsing of iso 8601 timestamps with zero microseconds (#282)
+
 Version 0.15
 ------------
 

attic/archive.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta, timezone
+from datetime import datetime
 from getpass import getuser
 from itertools import groupby
 import errno
@@ -14,11 +14,10 @@ import sys
 import time
 from io import BytesIO
 from attic import xattr
-from attic.cache import Cache
 from attic.platform import acl_get, acl_set
 from attic.chunker import Chunker
 from attic.hashindex import ChunkIndex
-from attic.helpers import Error, uid2user, user2uid, gid2group, group2gid, \
+from attic.helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
     Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
 
 ITEMS_BUFFER = 1024 * 1024
@@ -173,11 +172,7 @@ class Archive:
     @property
     def ts(self):
         """Timestamp of archive creation in UTC"""
-        t = self.metadata[b'time'].split('.', 1)
-        dt = datetime.strptime(t[0], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
-        if len(t) > 1:
-            dt += timedelta(seconds=float('.' + t[1]))
-        return dt
+        return parse_timestamp(self.metadata[b'time'])
 
     def __repr__(self):
         return 'Archive(%r)' % self.name
@@ -224,22 +219,19 @@ class Archive:
         self.repository.commit()
         self.cache.commit()
 
-    def calc_stats(self):
+    def calc_stats(self, cache):
         def add(id):
             count, size, csize = cache.chunks[id]
             stats.update(size, csize, count == 1)
-            cache.chunks[id] = count - 1, size, csize  # dirties cache.chunks!
-
+            cache.chunks[id] = count - 1, size, csize
         def add_file_chunks(chunks):
             for id, _, _ in chunks:
                 add(id)
-
         # This function is a bit evil since it abuses the cache to calculate
-        # the stats. The cache transaction must be rolled back afterwards.
-        cache = Cache(self.repository, self.key, self.manifest)
+        # the stats. The cache transaction must be rolled back afterwards
+        unpacker = msgpack.Unpacker(use_list=False)
         cache.begin_txn()
         stats = Statistics()
-        unpacker = msgpack.Unpacker(use_list=False)
         add(self.id)
         for id, chunk in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):
             add(id)

attic/archiver.py

@@ -361,7 +361,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         manifest, key = Manifest.load(repository)
         cache = Cache(repository, key, manifest, do_files=args.cache_files)
         archive = Archive(repository, key, manifest, args.archive.archive, cache=cache)
-        stats = archive.calc_stats()
+        stats = archive.calc_stats(cache)
         print('Name:', archive.name)
         print('Fingerprint: %s' % hexlify(archive.id).decode('ascii'))
         print('Hostname:', archive.metadata[b'hostname'])

attic/cache.py

@@ -103,11 +103,7 @@ class Cache:
         os.remove(os.path.join(self.path, 'config'))  # kill config first
         shutil.rmtree(self.path)
 
-    def open(self):
-        if not os.path.isdir(self.path):
-            raise Exception('%s Does not look like an Attic cache' % self.path)
-        self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True)
-        self.rollback()
+    def _do_open(self):
         self.config = RawConfigParser()
         self.config.read(os.path.join(self.path, 'config'))
         if self.config.getint('cache', 'version') != 1:
@@ -121,6 +117,12 @@ class Cache:
                                       key_size=self.repository.key_size)
         self.files = None
 
+    def open(self):
+        if not os.path.isdir(self.path):
+            raise Exception('%s Does not look like an Attic cache' % self.path)
+        self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True)
+        self.rollback()
+
     def close(self):
         if self.lock:
             self.lock.release()
@@ -193,6 +195,7 @@ class Cache:
             if os.path.exists(os.path.join(self.path, 'txn.tmp')):
                 shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
         self.txn_active = False
+        self._do_open()
 
     def sync(self):
         """Initializes cache by fetching and reading all archive indicies

attic/helpers.py

@@ -199,6 +199,14 @@ def to_localtime(ts):
     return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6])
 
 
+def parse_timestamp(timestamp):
+    """Parse a ISO 8601 timestamp string"""
+    if '.' in timestamp:  # microseconds might not be pressent
+        return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc)
+    else:
+        return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
+
+
 def update_excludes(args):
     """Merge exclude patterns from files with those on command line.
     Empty lines and lines starting with '#' are ignored, but whitespace

attic/key.py

@@ -36,6 +36,11 @@ class UnsupportedPayloadError(Error):
     """Unsupported payload type {}. A newer version is required to access this repository.
     """
 
+class KeyfileNotFoundError(Error):
+    """No key file for repository {} found in {}.
+    """
+
+
 class sha512_256(object):  # note: can't subclass sha512
     """sha512, but digest truncated to 256bit - faster than sha256 on 64bit platforms"""
     digestsize = digest_size = 32
@@ -564,7 +569,7 @@ class KeyfileKey(AESKeyBase):
                 line = fd.readline().strip()
                 if line and line.startswith(cls.FILE_ID) and line[10:] == id:
                     return filename
-        raise Exception('Key file for repository with ID %s not found' % id)
+        raise KeyfileNotFoundError(repository._location.canonical_path(), get_keys_dir())
 
     def load(self, filename, passphrase):
         with open(filename, 'r') as fd:

attic/platform.py

@@ -1,12 +1,10 @@
-import os
+import sys
 
-platform = os.uname()[0]
-
-if platform == 'Linux':
+if sys.platform.startswith('linux'):
     from attic.platform_linux import acl_get, acl_set, API_VERSION
-elif platform == 'FreeBSD':
+elif sys.platform.startswith('freebsd'):
     from attic.platform_freebsd import acl_get, acl_set, API_VERSION
-elif platform == 'Darwin':
+elif sys.platform == 'darwin':
     from attic.platform_darwin import acl_get, acl_set, API_VERSION
 else:
     API_VERSION = 2

attic/testsuite/helpers.py

@@ -4,8 +4,8 @@ from datetime import datetime, timezone, timedelta
 import os
 import tempfile
 import unittest
-from attic.helpers import adjust_patterns, exclude_path, Location, format_timedelta, ExcludePattern, make_path_safe, UpgradableLock, prune_within, prune_split, \
-    StableDict, int_to_bigint, bigint_to_int
+from attic.helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, UpgradableLock, prune_within, prune_split, to_localtime, \
+    StableDict, int_to_bigint, bigint_to_int, parse_timestamp
 from attic.testsuite import AtticTestCase
 import msgpack
 
@@ -209,3 +209,10 @@ class StableDictTestCase(AtticTestCase):
         d = StableDict(foo=1, bar=2, boo=3, baz=4)
         self.assert_equal(list(d.items()), [('bar', 2), ('baz', 4), ('boo', 3), ('foo', 1)])
         self.assert_equal(hashlib.md5(msgpack.packb(d)).hexdigest(), 'fc78df42cd60691b3ac3dd2a2b39903f')
+
+
+class TestParseTimestamp(AtticTestCase):
+
+    def test(self):
+        self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc))
+        self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc))

setup.py

@@ -9,8 +9,6 @@ versioneer.versionfile_build = 'attic/_version.py'
 versioneer.tag_prefix = ''
 versioneer.parentdir_prefix = 'Attic-'  # dirname like 'myproject-1.2.0'
 
-platform = os.uname()[0]
-
 min_python = (3, 2)
 if sys.version_info < min_python:
     print("Attic requires Python %d.%d or later" % min_python)
@@ -89,11 +87,11 @@ ext_modules = [
     Extension('attic.chunker', [chunker_source]),
     Extension('attic.hashindex', [hashindex_source])
 ]
-if platform == 'Linux':
+if sys.platform.startswith('linux'):
     ext_modules.append(Extension('attic.platform_linux', [platform_linux_source], libraries=['acl']))
-elif platform == 'FreeBSD':
+elif sys.platform.startswith('freebsd'):
     ext_modules.append(Extension('attic.platform_freebsd', [platform_freebsd_source]))
-elif platform == 'Darwin':
+elif sys.platform == 'darwin':
     ext_modules.append(Extension('attic.platform_darwin', [platform_darwin_source]))
 
 # msgpack pure python data corruption was fixed in 0.4.6.