Merge remote-tracking branch 'origin/master' into logging-refactor

Conflicts:
	borg/archiver.py

Vagrantfile

@@ -1,11 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-# Automated creation of testing environments on misc. platforms
-# Usage:
-#   vagrant up OS
-#   vagrant ssh OS -c command
-#   vagrant halt OS
+# Automated creation of testing environments / binaries on misc. platforms
 
 def packages_prepare_wheezy
   return <<-EOF
@@ -28,6 +24,8 @@ def packages_debianoid
     apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
     apt-get install -y fakeroot build-essential git
     apt-get install -y python3-dev python3-setuptools
+    # for building python:
+    apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev
     # this way it works on older dists (like ubuntu 12.04) also:
     easy_install3 pip
     pip3 install virtualenv
@@ -349,14 +347,31 @@ Vagrant.configure(2) do |config|
   end
 
   config.vm.define "wheezy32" do |b|
-    b.vm.box = "puppetlabs/debian-7.8-32-nocm"
+    b.vm.box = "boxcutter/debian79-i386"
     b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("wheezy32")
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32")
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy32")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy32")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32")
   end
 
+  config.vm.define "wheezy64" do |b|
+    b.vm.box = "boxcutter/debian79"
+    b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy64")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy64")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64")
+  end
+
   # OS X
   config.vm.define "darwin64" do |b|
     b.vm.box = "jhcook/yosemite-clitools"

borg/archiver.py

@@ -20,6 +20,7 @@ import traceback
 from . import __version__
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
 from .compress import Compressor, COMPR_BUFFER
+from .upgrader import AtticRepositoryUpgrader
 from .repository import Repository
 from .cache import Cache
 from .key import key_creator
@@ -311,18 +312,20 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
             if args.stats:
                 logger.info(stats.print_('Deleted data:', cache))
         else:
-            print("You requested to completely DELETE the repository *including* all archives it contains:", file=sys.stderr)
-            for archive_info in manifest.list_archive_infos(sort_by='ts'):
-                print(format_archive(archive_info), file=sys.stderr)
-            if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
-                print("""Type "YES" if you understand this and want to continue.""", file=sys.stderr)
-                # XXX: prompt may end up on stdout, but we'll assume that input() does the right thing
-                if input('Do you want to continue? ') != 'YES':
-                    self.exit_code = 1
-                    return self.exit_code
-            repository.destroy()
+            if not args.cache_only:
+                print("You requested to completely DELETE the repository *including* all archives it contains:", file=sys.stderr)
+                for archive_info in manifest.list_archive_infos(sort_by='ts'):
+                    print(format_archive(archive_info), file=sys.stderr)
+                if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
+                    print("""Type "YES" if you understand this and want to continue.\n""", file=sys.stderr)
+                    # XXX: prompt may end up on stdout, but we'll assume that input() does the right thing
+                    if input('Do you want to continue? ') != 'YES':
+                        self.exit_code = 1
+                        return self.exit_code
+                repository.destroy()
+                logger.info("Repository deleted.")
             cache.destroy()
-            logger.info("Repository and corresponding cache were deleted.")
+            logger.info("Cache deleted.")
         return self.exit_code
 
     def do_mount(self, args):
@@ -456,6 +459,24 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
             logger.info(stats.print_('Deleted data:', cache))
         return self.exit_code
 
+    def do_upgrade(self, args):
+        """upgrade a repository from a previous version"""
+        # XXX: currently only upgrades from Attic repositories, but may
+        # eventually be extended to deal with major upgrades for borg
+        # itself.
+        #
+        # in this case, it should auto-detect the current repository
+        # format and fire up necessary upgrade mechanism. this remains
+        # to be implemented.
+
+        # XXX: should auto-detect if it is an attic repository here
+        repo = AtticRepositoryUpgrader(args.repository.path, create=False)
+        try:
+            repo.upgrade(args.dry_run)
+        except NotImplementedError as e:
+            print("warning: %s" % e)
+        return self.exit_code
+
     helptext = {}
     helptext['patterns'] = '''
         Exclude patterns use a variant of shell pattern syntax, with '*' matching any
@@ -800,6 +821,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         subparser.add_argument('-s', '--stats', dest='stats',
                                action='store_true', default=False,
                                help='print statistics for the deleted archive')
+        subparser.add_argument('-c', '--cache-only', dest='cache_only',
+                               action='store_true', default=False,
+                               help='delete only the local cache for the given repository')
         subparser.add_argument('target', metavar='TARGET', nargs='?', default='',
                                type=location_validator(),
                                help='archive or repository to delete')
@@ -904,6 +928,53 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                type=location_validator(archive=False),
                                help='repository to prune')
 
+        upgrade_epilog = textwrap.dedent("""
+        upgrade an existing Borg repository in place. this currently
+        only support converting an Attic repository, but may
+        eventually be extended to cover major Borg upgrades as well.
+
+        it will change the magic strings in the repository's segments
+        to match the new Borg magic strings. the keyfiles found in
+        $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and
+        copied to $BORG_KEYS_DIR or ~/.borg/keys.
+
+        the cache files are converted, from $ATTIC_CACHE_DIR or
+        ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the
+        cache layout between Borg and Attic changed, so it is possible
+        the first backup after the conversion takes longer than expected
+        due to the cache resync.
+
+        it is recommended you run this on a copy of the Attic
+        repository, in case something goes wrong, for example:
+
+            cp -a attic borg
+            borg upgrade -n borg
+            borg upgrade borg
+
+        upgrade should be able to resume if interrupted, although it
+        will still iterate over all segments. if you want to start
+        from scratch, use `borg delete` over the copied repository to
+        make sure the cache files are also removed:
+
+            borg delete borg
+
+        the conversion can PERMANENTLY DAMAGE YOUR REPOSITORY! Attic
+        will also NOT BE ABLE TO READ THE BORG REPOSITORY ANYMORE, as
+        the magic strings will have changed.
+
+        you have been warned.""")
+        subparser = subparsers.add_parser('upgrade', parents=[common_parser],
+                                          description=self.do_upgrade.__doc__,
+                                          epilog=upgrade_epilog,
+                                          formatter_class=argparse.RawDescriptionHelpFormatter)
+        subparser.set_defaults(func=self.do_upgrade)
+        subparser.add_argument('-n', '--dry-run', dest='dry_run',
+                               default=False, action='store_true',
+                               help='do not change repository')
+        subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False),
+                               help='path to the repository to be upgraded')
+
         subparser = subparsers.add_parser('help', parents=[common_parser],
                                           description='Extra help')
         subparser.add_argument('--epilog-only', dest='epilog_only',

borg/cache.py

@@ -1,4 +1,4 @@
-from configparser import RawConfigParser
+import configparser
 from .remote import cache_if_remote
 from collections import namedtuple
 import errno
@@ -108,7 +108,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}""")
         os.makedirs(self.path)
         with open(os.path.join(self.path, 'README'), 'w') as fd:
             fd.write('This is a Borg cache')
-        config = RawConfigParser()
+        config = configparser.RawConfigParser()
         config.add_section('cache')
         config.set('cache', 'version', '1')
         config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii'))
@@ -128,10 +128,17 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}""")
         shutil.rmtree(self.path)
 
     def _do_open(self):
-        self.config = RawConfigParser()
-        self.config.read(os.path.join(self.path, 'config'))
-        if self.config.getint('cache', 'version') != 1:
-            raise Exception('%s Does not look like a Borg cache')
+        self.config = configparser.RawConfigParser()
+        config_path = os.path.join(self.path, 'config')
+        self.config.read(config_path)
+        try:
+            cache_version = self.config.getint('cache', 'version')
+            wanted_version = 1
+            if  cache_version != wanted_version:
+                raise Exception('%s has unexpected cache version %d (wanted: %d).' % (
+                    config_path, cache_version, wanted_version))
+        except configparser.NoSectionError as e:
+            raise Exception('%s does not look like a Borg cache.' % config_path)
         self.id = self.config.get('cache', 'repository')
         self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
         self.timestamp = self.config.get('cache', 'timestamp', fallback=None)
@@ -238,9 +245,12 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}""")
             return path.encode('utf-8')
 
         def cached_archives():
-            fns = os.listdir(archive_path)
-            # filenames with 64 hex digits == 256bit
-            return set(unhexlify(fn) for fn in fns if len(fn) == 64)
+            if self.do_cache:
+                fns = os.listdir(archive_path)
+                # filenames with 64 hex digits == 256bit
+                return set(unhexlify(fn) for fn in fns if len(fn) == 64)
+            else:
+                return set()
 
         def repo_archives():
             return set(info[b'id'] for info in self.manifest.archives.values())
@@ -277,14 +287,15 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}""")
                     if b'chunks' in item:
                         for chunk_id, size, csize in item[b'chunks']:
                             add(chunk_idx, chunk_id, size, csize)
-            fn = mkpath(archive_id)
-            fn_tmp = mkpath(archive_id, suffix='.tmp')
-            try:
-                chunk_idx.write(fn_tmp)
-            except Exception:
-                os.unlink(fn_tmp)
-            else:
-                os.rename(fn_tmp, fn)
+            if self.do_cache:
+                fn = mkpath(archive_id)
+                fn_tmp = mkpath(archive_id, suffix='.tmp')
+                try:
+                    chunk_idx.write(fn_tmp)
+                except Exception:
+                    os.unlink(fn_tmp)
+                else:
+                    os.rename(fn_tmp, fn)
             return chunk_idx
 
         def lookup_name(archive_id):
@@ -342,6 +353,9 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}""")
         self.begin_txn()
         repository = cache_if_remote(self.repository)
         legacy_cleanup()
+        # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d -
+        # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk)
+        self.do_cache = os.path.isdir(archive_path)
         self.chunks = create_master_idx(self.chunks)
 
     def add_chunk(self, id, data, stats):

borg/testsuite/upgrader.py

@@ -0,0 +1,163 @@
+import os
+import shutil
+import tempfile
+
+import pytest
+
+try:
+    import attic.repository
+    import attic.key
+    import attic.helpers
+except ImportError:
+    attic = None
+
+from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
+from ..helpers import get_keys_dir
+from ..key import KeyfileKey
+from ..repository import Repository, MAGIC
+
+pytestmark = pytest.mark.skipif(attic is None,
+                                reason='cannot find an attic install')
+
+
+def repo_valid(path):
+    """
+    utility function to check if borg can open a repository
+
+    :param path: the path to the repository
+    :returns: if borg can check the repository
+    """
+    repository = Repository(str(path), create=False)
+    # can't check raises() because check() handles the error
+    state = repository.check()
+    repository.close()
+    return state
+
+
+def key_valid(path):
+    """
+    check that the new keyfile is alright
+
+    :param path: the path to the key file
+    :returns: if the file starts with the borg magic string
+    """
+    keyfile = os.path.join(get_keys_dir(),
+                           os.path.basename(path))
+    with open(keyfile, 'r') as f:
+        return f.read().startswith(KeyfileKey.FILE_ID)
+
+
+@pytest.fixture()
+def attic_repo(tmpdir):
+    """
+    create an attic repo with some stuff in it
+
+    :param tmpdir: path to the repository to be created
+    :returns: a attic.repository.Repository object
+    """
+    attic_repo = attic.repository.Repository(str(tmpdir), create=True)
+    # throw some stuff in that repo, copied from `RepositoryTestCase.test1`
+    for x in range(100):
+        attic_repo.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
+    attic_repo.commit()
+    attic_repo.close()
+    return attic_repo
+
+
+def test_convert_segments(tmpdir, attic_repo):
+    """test segment conversion
+
+    this will load the given attic repository, list all the segments
+    then convert them one at a time. we need to close the repo before
+    conversion otherwise we have errors from borg
+
+    :param tmpdir: a temporary directory to run the test in (builtin
+    fixture)
+    :param attic_repo: a populated attic repository (fixture)
+    """
+    # check should fail because of magic number
+    assert not repo_valid(tmpdir)
+    print("opening attic repository with borg and converting")
+    repo = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    segments = [filename for i, filename in repo.io.segment_iterator()]
+    repo.close()
+    repo.convert_segments(segments, dryrun=False)
+    repo.convert_cache(dryrun=False)
+    assert repo_valid(tmpdir)
+
+
+class MockArgs:
+    """
+    mock attic location
+
+    this is used to simulate a key location with a properly loaded
+    repository object to create a key file
+    """
+    def __init__(self, path):
+        self.repository = attic.helpers.Location(path)
+
+
+@pytest.fixture()
+def attic_key_file(attic_repo, tmpdir):
+    """
+    create an attic key file from the given repo, in the keys
+    subdirectory of the given tmpdir
+
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :returns: the KeyfileKey object as returned by
+    attic.key.KeyfileKey.create()
+    """
+    keys_dir = str(tmpdir.mkdir('keys'))
+
+    # we use the repo dir for the created keyfile, because we do
+    # not want to clutter existing keyfiles
+    os.environ['ATTIC_KEYS_DIR'] = keys_dir
+
+    # we use the same directory for the converted files, which
+    # will clutter the previously created one, which we don't care
+    # about anyways. in real runs, the original key will be retained.
+    os.environ['BORG_KEYS_DIR'] = keys_dir
+    os.environ['ATTIC_PASSPHRASE'] = 'test'
+    return attic.key.KeyfileKey.create(attic_repo,
+                                       MockArgs(keys_dir))
+
+
+def test_keys(tmpdir, attic_repo, attic_key_file):
+    """test key conversion
+
+    test that we can convert the given key to a properly formatted
+    borg key. assumes that the ATTIC_KEYS_DIR and BORG_KEYS_DIR have
+    been properly populated by the attic_key_file fixture.
+
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    """
+    repository = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    keyfile = AtticKeyfileKey.find_key_file(repository)
+    AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False)
+    assert key_valid(attic_key_file.path)
+
+
+def test_convert_all(tmpdir, attic_repo, attic_key_file):
+    """test all conversion steps
+
+    this runs everything. mostly redundant test, since everything is
+    done above. yet we expect a NotImplementedError because we do not
+    convert caches yet.
+
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    """
+    # check should fail because of magic number
+    assert not repo_valid(tmpdir)
+    print("opening attic repository with borg and converting")
+    repo = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    repo.upgrade(dryrun=False)
+    assert key_valid(attic_key_file.path)
+    assert repo_valid(tmpdir)

borg/upgrader.py

@@ -0,0 +1,233 @@
+from binascii import hexlify
+import os
+import shutil
+import time
+
+from .helpers import get_keys_dir, get_cache_dir
+from .locking import UpgradableLock
+from .repository import Repository, MAGIC
+from .key import KeyfileKey, KeyfileNotFoundError
+
+ATTIC_MAGIC = b'ATTICSEG'
+
+
+class AtticRepositoryUpgrader(Repository):
+    def upgrade(self, dryrun=True):
+        """convert an attic repository to a borg repository
+
+        those are the files that need to be upgraded here, from most
+        important to least important: segments, key files, and various
+        caches, the latter being optional, as they will be rebuilt if
+        missing.
+
+        we nevertheless do the order in reverse, as we prefer to do
+        the fast stuff first, to improve interactivity.
+        """
+        print("reading segments from attic repository using borg")
+        # we need to open it to load the configuration and other fields
+        self.open(self.path, exclusive=False)
+        segments = [filename for i, filename in self.io.segment_iterator()]
+        try:
+            keyfile = self.find_attic_keyfile()
+        except KeyfileNotFoundError:
+            print("no key file found for repository")
+        else:
+            self.convert_keyfiles(keyfile, dryrun)
+        self.close()
+        # partial open: just hold on to the lock
+        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
+                                   exclusive=True).acquire()
+        try:
+            self.convert_cache(dryrun)
+            self.convert_segments(segments, dryrun)
+        finally:
+            self.lock.release()
+            self.lock = None
+
+    @staticmethod
+    def convert_segments(segments, dryrun):
+        """convert repository segments from attic to borg
+
+        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
+        `$ATTIC_REPO/data/**`.
+
+        luckily the magic string length didn't change so we can just
+        replace the 8 first bytes of all regular files in there."""
+        print("converting %d segments..." % len(segments))
+        i = 0
+        for filename in segments:
+            i += 1
+            print("\rconverting segment %d/%d in place, %.2f%% done (%s)"
+                  % (i, len(segments), 100*float(i)/len(segments), filename), end='')
+            if dryrun:
+                time.sleep(0.001)
+            else:
+                AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC)
+        print()
+
+    @staticmethod
+    def header_replace(filename, old_magic, new_magic):
+        with open(filename, 'r+b') as segment:
+            segment.seek(0)
+            # only write if necessary
+            if segment.read(len(old_magic)) == old_magic:
+                segment.seek(0)
+                segment.write(new_magic)
+
+    def find_attic_keyfile(self):
+        """find the attic keyfiles
+
+        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
+        finds the keys with the right identifier for the repo.
+
+        this is expected to look into $HOME/.attic/keys or
+        $ATTIC_KEYS_DIR for key files matching the given Borg
+        repository.
+
+        it is expected to raise an exception (KeyfileNotFoundError) if
+        no key is found. whether that exception is from Borg or Attic
+        is unclear.
+
+        this is split in a separate function in case we want to use
+        the attic code here directly, instead of our local
+        implementation."""
+        return AtticKeyfileKey.find_key_file(self)
+
+    @staticmethod
+    def convert_keyfiles(keyfile, dryrun):
+
+        """convert key files from attic to borg
+
+        replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
+        `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
+        `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
+        `$HOME/.borg/keys`.
+
+        no need to decrypt to convert. we need to rewrite the whole
+        key file because magic string length changed, but that's not a
+        problem because the keyfiles are small (compared to, say,
+        all the segments)."""
+        print("converting keyfile %s" % keyfile)
+        with open(keyfile, 'r') as f:
+            data = f.read()
+        data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1)
+        keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile))
+        print("writing borg keyfile to %s" % keyfile)
+        if not dryrun:
+            with open(keyfile, 'w') as f:
+                f.write(data)
+
+    def convert_cache(self, dryrun):
+        """convert caches from attic to borg
+
+        those are all hash indexes, so we need to
+        `s/ATTICIDX/BORG_IDX/` in a few locations:
+
+        * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
+          is the `Repository.get_index_transaction_id()`), which we
+          should probably update, with a lock, see
+          `Repository.open()`, which i'm not sure we should use
+          because it may write data on `Repository.close()`...
+
+        * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or
+          `$HOME/.cache/attic/<repoid>/`), which we could just drop,
+          but if we'd want to convert, we could open it with the
+          `Cache.open()`, edit in place and then `Cache.close()` to
+          make sure we have locking right
+        """
+        caches = []
+        transaction_id = self.get_index_transaction_id()
+        if transaction_id is None:
+            print('no index file found for repository %s' % self.path)
+        else:
+            caches += [os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8')]
+
+        # copy of attic's get_cache_dir()
+        attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
+                                         os.path.join(os.path.expanduser('~'),
+                                                      '.cache', 'attic'))
+        attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii'))
+        borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii'))
+
+        def copy_cache_file(path):
+            """copy the given attic cache path into the borg directory
+
+            does nothing if dryrun is True. also expects
+            attic_cache_dir and borg_cache_dir to be set in the parent
+            scope, to the directories path including the repository
+            identifier.
+
+            :params path: the basename of the cache file to copy
+            (example: "files" or "chunks") as a string
+
+            :returns: the borg file that was created or None if non
+            was created.
+
+            """
+            attic_file = os.path.join(attic_cache_dir, path)
+            if os.path.exists(attic_file):
+                borg_file = os.path.join(borg_cache_dir, path)
+                if os.path.exists(borg_file):
+                    print("borg cache file already exists in %s, skipping conversion of %s" % (borg_file, attic_file))
+                else:
+                    print("copying attic cache file from %s to %s" % (attic_file, borg_file))
+                    if not dryrun:
+                        shutil.copyfile(attic_file, borg_file)
+                    return borg_file
+            else:
+                print("no %s cache file found in %s" % (path, attic_file))
+            return None
+
+        # XXX: untested, because generating cache files is a PITA, see
+        # Archiver.do_create() for proof
+        if os.path.exists(attic_cache_dir):
+            if not os.path.exists(borg_cache_dir):
+                os.makedirs(borg_cache_dir)
+
+            # file that we don't have a header to convert, just copy
+            for cache in ['config', 'files']:
+                copy_cache_file(cache)
+
+            # we need to convert the headers of those files, copy first
+            for cache in ['chunks']:
+                copied = copy_cache_file(cache)
+                if copied:
+                    print("converting cache %s" % cache)
+                    if not dryrun:
+                        AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX')
+
+
+class AtticKeyfileKey(KeyfileKey):
+    """backwards compatible Attic key file parser"""
+    FILE_ID = 'ATTIC KEY'
+
+    # verbatim copy from attic
+    @staticmethod
+    def get_keys_dir():
+        """Determine where to repository keys and cache"""
+        return os.environ.get('ATTIC_KEYS_DIR',
+                              os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
+
+    @classmethod
+    def find_key_file(cls, repository):
+        """copy of attic's `find_key_file`_
+
+        this has two small modifications:
+
+        1. it uses the above `get_keys_dir`_ instead of the global one,
+           assumed to be borg's
+
+        2. it uses `repository.path`_ instead of
+           `repository._location.canonical_path`_ because we can't
+           assume the repository has been opened by the archiver yet
+        """
+        get_keys_dir = cls.get_keys_dir
+        id = hexlify(repository.id).decode('ascii')
+        keys_dir = get_keys_dir()
+        for name in os.listdir(keys_dir):
+            filename = os.path.join(keys_dir, name)
+            with open(filename, 'r') as fd:
+                line = fd.readline().strip()
+                if line and line.startswith(cls.FILE_ID) and line[10:] == id:
+                    return filename
+        raise KeyfileNotFoundError(repository.path, get_keys_dir())

docs/Makefile

@@ -36,7 +36,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html: usage
+html: usage api.rst
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -153,3 +153,18 @@ usage/%.rst.inc: ../borg/archiver.py
 	@borg help $* --usage-only | sed -e 's/^/    /' >> $@
 	@printf "\nDescription\n~~~~~~~~~~~\n" >> $@
 	@borg help $* --epilog-only >> $@
+
+api.rst: Makefile
+	@echo "auto-generating API documentation"
+	@echo "Borg Backup API documentation" > $@
+	@echo "=============================" >> $@
+	@echo "" >> $@
+	@for mod in ../borg/*.pyx ../borg/*.py; do \
+		if echo "$$mod" | grep -q "/_"; then \
+			continue ; \
+		fi ; \
+		printf ".. automodule:: "; \
+		echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \
+		echo "    :members:"; \
+		echo "    :undoc-members:"; \
+	done >> $@

docs/conf.py

@@ -218,7 +218,7 @@ latex_documents = [
 #     ['see "AUTHORS" file'], 1)
 #]
 
-extensions = ['sphinx.ext.extlinks']
+extensions = ['sphinx.ext.extlinks', 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode']
 
 extlinks = {
     'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),

docs/index.rst

@@ -16,3 +16,4 @@ Borg Documentation
    changes
    internals
    development
+   api

docs/quickstart.rst

@@ -85,9 +85,12 @@ certain number of old archives::
         --exclude /home/Ben/Music/Justin\ Bieber    \
         --exclude '*.pyc'
 
-    # Use the `prune` subcommand to maintain 7 daily, 4 weekly
-    # and 6 monthly archives.
-    borg prune -v $REPOSITORY --keep-daily=7 --keep-weekly=4 --keep-monthly=6
+    # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
+    # archives of THIS machine. --prefix `hostname`- is very important to
+    # limit prune's operation to this machine's archives and not apply to
+    # other machine's archives also.
+    borg prune -v $REPOSITORY --prefix `hostname`- \
+        --keep-daily=7 --keep-weekly=4 --keep-monthly=6
 
 .. backup_compression:
 

docs/usage.rst

@@ -265,10 +265,23 @@ Examples
 
 Examples
 ~~~~~~~~
+
+Be careful, prune is potentially dangerous command, it will remove backup
+archives.
+
+The default of prune is to apply to **all archives in the repository** unless
+you restrict its operation to a subset of the archives using `--prefix`.
+When using --prefix, be careful to choose a good prefix - e.g. do not use a
+prefix "foo" if you do not also want to match "foobar".
+
+It is strongly recommended to always run `prune --dry-run ...` first so you
+will see what it would do without it actually doing anything.
+
 ::
 
-    # Keep 7 end of day and 4 additional end of week archives:
-    $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4
+    # Keep 7 end of day and 4 additional end of week archives.
+    # Do a dry-run without actually deleting anything.
+    $ borg prune /mnt/backup --dry-run --keep-daily=7 --keep-weekly=4
 
     # Same as above but only apply to archive names starting with "foo":
     $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --prefix=foo

tox.ini

@@ -2,13 +2,15 @@
 # fakeroot -u tox --recreate
 
 [tox]
-envlist = py32, py33, py34, py35
+envlist = py{32,33,34,35}
 
 [testenv]
 # Change dir to avoid import problem for cython code. The directory does
 # not really matter, should be just different from the toplevel dir.
 changedir = {toxworkdir}
-deps = -rrequirements.d/development.txt
+deps =
+     -rrequirements.d/development.txt
+     attic
 commands = py.test --cov=borg --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *