Merge pull request #505 from ThomasWaldmann/prep-rel1

Prepare release 1.0

.travis.yml

@@ -8,26 +8,12 @@ cache:
 
 matrix:
     include:
-        - python: 3.2
-          os: linux
-          env: TOXENV=py32
-        - python: 3.3
-          os: linux
-          env: TOXENV=py33
         - python: 3.4
           os: linux
           env: TOXENV=py34
         - python: 3.5
           os: linux
           env: TOXENV=py35
-        - language: generic
-          os: osx
-          osx_image: xcode6.4
-          env: TOXENV=py32
-        - language: generic
-          os: osx
-          osx_image: xcode6.4
-          env: TOXENV=py33
         - language: generic
           os: osx
           osx_image: xcode6.4

.travis/install.sh

@@ -18,21 +18,13 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
     brew outdated pyenv || brew upgrade pyenv
 
     case "${TOXENV}" in
-        py32)
-            pyenv install 3.2.6
-            pyenv global 3.2.6
-            ;;
-        py33)
-            pyenv install 3.3.6
-            pyenv global 3.3.6
-            ;;
         py34)
             pyenv install 3.4.3
             pyenv global 3.4.3
             ;;
         py35)
-            pyenv install 3.5.0
-            pyenv global 3.5.0
+            pyenv install 3.5.1
+            pyenv global 3.5.1
             ;;
     esac
     pyenv rehash

Vagrantfile

@@ -161,7 +161,6 @@ end
 def install_pythons(boxname)
   return <<-EOF
     . ~/.bash_profile
-    pyenv install 3.3.0  # tests
     pyenv install 3.4.0  # tests
     pyenv install 3.5.0  # tests
     pyenv install 3.5.1  # binary build, use latest 3.5.x release
@@ -251,7 +250,7 @@ def run_tests(boxname)
     . ../borg-env/bin/activate
     if which pyenv > /dev/null; then
       # for testing, use the earliest point releases of the supported python versions:
-      pyenv global 3.3.0 3.4.0 3.5.0
+      pyenv global 3.4.0 3.5.0
     fi
     # otherwise: just use the system python
     if which fakeroot > /dev/null; then

borg/_chunker.c

@@ -186,19 +186,6 @@ chunker_fill(Chunker *c)
     return 1;
 }
 
-static PyObject *
-PyBuffer_FromMemory(void *data, Py_ssize_t len)
-{
-    Py_buffer buffer;
-    PyObject *mv;
-
-    PyBuffer_FillInfo(&buffer, NULL, data, len, 1, PyBUF_CONTIG_RO);
-    mv = PyMemoryView_FromBuffer(&buffer);
-    PyBuffer_Release(&buffer);
-    return mv;
-}
-
-
 static PyObject *
 chunker_process(Chunker *c)
 {
@@ -221,7 +208,7 @@ chunker_process(Chunker *c)
         c->done = 1;
         if(c->remaining) {
             c->bytes_yielded += c->remaining;
-            return PyBuffer_FromMemory(c->data + c->position, c->remaining);
+            return PyMemoryView_FromMemory(c->data + c->position, c->remaining, PyBUF_READ);
         }
         else {
             if(c->bytes_read == c->bytes_yielded)
@@ -253,5 +240,5 @@ chunker_process(Chunker *c)
     c->last = c->position;
     n = c->last - old_last;
     c->bytes_yielded += n;
-    return PyBuffer_FromMemory(c->data + old_last, n);
+    return PyMemoryView_FromMemory(c->data + old_last, n, PyBUF_READ);
 }

borg/archive.py

@@ -18,7 +18,7 @@ from io import BytesIO
 from . import xattr
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \
     Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
-    st_atime_ns, st_ctime_ns, st_mtime_ns, ProgressIndicatorPercent
+    ProgressIndicatorPercent
 from .platform import acl_get, acl_set
 from .chunker import Chunker
 from .hashindex import ChunkIndex
@@ -26,10 +26,10 @@ import msgpack
 
 ITEMS_BUFFER = 1024 * 1024
 
-CHUNK_MIN_EXP = 10  # 2**10 == 1kiB
+CHUNK_MIN_EXP = 19  # 2**19 == 512kiB
 CHUNK_MAX_EXP = 23  # 2**23 == 8MiB
 HASH_WINDOW_SIZE = 0xfff  # 4095B
-HASH_MASK_BITS = 16  # results in ~64kiB chunks statistically
+HASH_MASK_BITS = 21  # results in ~2MiB chunks statistically
 
 # defaults, use --chunker-params to override
 CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)
@@ -37,18 +37,9 @@ CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE
 # chunker params for the items metadata stream, finer granularity
 ITEMS_CHUNKER_PARAMS = (12, 16, 14, HASH_WINDOW_SIZE)
 
-utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
-utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {})
-has_mtime_ns = sys.version >= '3.3'
 has_lchmod = hasattr(os, 'lchmod')
 has_lchflags = hasattr(os, 'lchflags')
 
-# Python <= 3.2 raises OSError instead of PermissionError (See #164)
-try:
-    PermissionError = PermissionError
-except NameError:
-    PermissionError = OSError
-
 
 class DownloadPipeline:
 
@@ -301,7 +292,7 @@ Number of files: {0.stats.nfiles}'''.format(self)
             else:
                 os.unlink(path)
         except UnicodeEncodeError:
-            raise self.IncompatibleFilesystemEncodingError(path, sys.getfilesystemencoding())
+            raise self.IncompatibleFilesystemEncodingError(path, sys.getfilesystemencoding()) from None
         except OSError:
             pass
         mode = item[b'mode']
@@ -341,7 +332,7 @@ Number of files: {0.stats.nfiles}'''.format(self)
             try:
                 os.symlink(source, path)
             except UnicodeEncodeError:
-                raise self.IncompatibleFilesystemEncodingError(source, sys.getfilesystemencoding())
+                raise self.IncompatibleFilesystemEncodingError(source, sys.getfilesystemencoding()) from None
             self.restore_attrs(path, item, symlink=True)
         elif stat.S_ISFIFO(mode):
             if not os.path.exists(os.path.dirname(path)):
@@ -392,12 +383,10 @@ Number of files: {0.stats.nfiles}'''.format(self)
         else:
             # old archives only had mtime in item metadata
             atime = mtime
-        if fd and utime_supports_fd:  # Python >= 3.3
+        if fd:
             os.utime(fd, None, ns=(atime, mtime))
-        elif utime_supports_follow_symlinks:  # Python >= 3.3
+        else:
             os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
-        elif not symlink:
-            os.utime(path, (atime / 1e9, mtime / 1e9))
         acl_set(path, item, self.numeric_owner)
         # Only available on OS X and FreeBSD
         if has_lchflags and b'bsdflags' in item:
@@ -441,9 +430,9 @@ Number of files: {0.stats.nfiles}'''.format(self)
             b'mode': st.st_mode,
             b'uid': st.st_uid, b'user': uid2user(st.st_uid),
             b'gid': st.st_gid, b'group': gid2group(st.st_gid),
-            b'atime': int_to_bigint(st_atime_ns(st)),
-            b'ctime': int_to_bigint(st_ctime_ns(st)),
-            b'mtime': int_to_bigint(st_mtime_ns(st)),
+            b'atime': int_to_bigint(st.st_atime_ns),
+            b'ctime': int_to_bigint(st.st_ctime_ns),
+            b'mtime': int_to_bigint(st.st_mtime_ns),
         }
         if self.numeric_owner:
             item[b'user'] = item[b'group'] = None

borg/archiver.py

@@ -1,10 +1,8 @@
-from .support import argparse  # see support/__init__.py docstring
-                               # DEPRECATED - remove after requiring py 3.4
-
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 from datetime import datetime
 from hashlib import sha256
 from operator import attrgetter
+import argparse
 import functools
 import inspect
 import io
@@ -17,8 +15,8 @@ import traceback
 
 from . import __version__
 from .helpers import Error, location_validator, format_time, format_file_size, \
-    format_file_mode, parse_pattern, PathPrefixPattern, to_localtime, timestamp, \
-    get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \
+    parse_pattern, PathPrefixPattern, to_localtime, timestamp, \
+    get_cache_dir, get_keys_dir, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
@@ -28,7 +26,7 @@ from .compress import Compressor, COMPR_BUFFER
 from .upgrader import AtticRepositoryUpgrader
 from .repository import Repository
 from .cache import Cache
-from .key import key_creator
+from .key import key_creator, RepoKey, PassphraseKey
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
 
@@ -66,7 +64,6 @@ class Archiver:
             repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args)
         else:
             repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait, lock=lock)
-        repository._location = location
         return repository
 
     def print_error(self, msg, *args):
@@ -108,8 +105,8 @@ class Archiver:
             msg = ("'check --repair' is an experimental feature that might result in data loss." +
                    "\n" +
                    "Type 'YES' if you understand this and want to continue: ")
-            if not yes(msg, false_msg="Aborting.", default_notty=False,
-                       env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )):
+            if not yes(msg, false_msg="Aborting.", truish=('YES', ),
+                       env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
                 return EXIT_ERROR
         if not args.archives_only:
             if not repository.check(repair=args.repair, save_space=args.save_space):
@@ -127,6 +124,23 @@ class Archiver:
         key.change_passphrase()
         return EXIT_SUCCESS
 
+    def do_migrate_to_repokey(self, args):
+        """Migrate passphrase -> repokey"""
+        repository = self.open_repository(args)
+        manifest_data = repository.get(Manifest.MANIFEST_ID)
+        key_old = PassphraseKey.detect(repository, manifest_data)
+        key_new = RepoKey(repository)
+        key_new.target = repository
+        key_new.repository_id = repository.id
+        key_new.enc_key = key_old.enc_key
+        key_new.enc_hmac_key = key_old.enc_hmac_key
+        key_new.id_key = key_old.id_key
+        key_new.chunk_seed = key_old.chunk_seed
+        key_new.change_passphrase()  # option to change key protection passphrase, save
+        return EXIT_SUCCESS
+
+        return EXIT_SUCCESS
+
     def do_create(self, args):
         """Create new archive"""
         matcher = PatternMatcher(fallback=True)
@@ -139,14 +153,14 @@ class Archiver:
             try:
                 st = os.stat(get_cache_dir())
                 skip_inodes.add((st.st_ino, st.st_dev))
-            except IOError:
+            except OSError:
                 pass
             # Add local repository dir to inode_skip list
             if not args.location.host:
                 try:
                     st = os.stat(args.location.path)
                     skip_inodes.add((st.st_ino, st.st_dev))
-                except IOError:
+                except OSError:
                     pass
             for path in args.paths:
                 if path == '-':  # stdin
@@ -154,7 +168,7 @@ class Archiver:
                     if not dry_run:
                         try:
                             status = archive.process_stdin(path, cache)
-                        except IOError as e:
+                        except OSError as e:
                             status = 'E'
                             self.print_warning('%s: %s', path, e)
                     else:
@@ -231,7 +245,7 @@ class Archiver:
             if not dry_run:
                 try:
                     status = archive.process_file(path, st, cache)
-                except IOError as e:
+                except OSError as e:
                     status = 'E'
                     self.print_warning('%s: %s', path, e)
         elif stat.S_ISDIR(st.st_mode):
@@ -328,7 +342,7 @@ class Archiver:
                         archive.extract_item(item, restore_attrs=False)
                     else:
                         archive.extract_item(item, stdout=stdout, sparse=sparse)
-            except IOError as e:
+            except OSError as e:
                 self.print_warning('%s: %s', remove_surrogates(orig_path), e)
 
         if not args.dry_run:
@@ -377,8 +391,8 @@ class Archiver:
                         msg.append(format_archive(archive_info))
                     msg.append("Type 'YES' if you understand this and want to continue: ")
                     msg = '\n'.join(msg)
-                    if not yes(msg, false_msg="Aborting.", default_notty=False,
-                               env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )):
+                    if not yes(msg, false_msg="Aborting.", truish=('YES', ),
+                               env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'):
                         self.exit_code = EXIT_ERROR
                         return self.exit_code
                     repository.destroy()
@@ -428,10 +442,9 @@ class Archiver:
                 for item in archive.iter_items():
                     print(remove_surrogates(item[b'path']))
             else:
-                tmap = {1: 'p', 2: 'c', 4: 'd', 6: 'b', 0o10: '-', 0o12: 'l', 0o14: 's'}
                 for item in archive.iter_items():
-                    type = tmap.get(item[b'mode'] // 4096, '?')
-                    mode = format_file_mode(item[b'mode'])
+                    mode = stat.filemode(item[b'mode'])
+                    type = mode[0]
                     size = 0
                     if type == '-':
                         try:
@@ -440,19 +453,19 @@ class Archiver:
                             pass
                     try:
                         mtime = datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9)
-                    except ValueError:
+                    except OverflowError:
                         # likely a broken mtime and datetime did not want to go beyond year 9999
                         mtime = datetime(9999, 12, 31, 23, 59, 59)
                     if b'source' in item:
                         if type == 'l':
                             extra = ' -> %s' % item[b'source']
                         else:
-                            type = 'h'
+                            mode = 'h' + mode[1:]
                             extra = ' link to %s' % item[b'source']
                     else:
                         extra = ''
-                    print('%s%s %-6s %-6s %8d %s %s%s' % (
-                        type, mode, item[b'user'] or item[b'uid'],
+                    print('%s %-6s %-6s %8d %s %s%s' % (
+                        mode, item[b'user'] or item[b'uid'],
                         item[b'group'] or item[b'gid'], size, format_time(mtime),
                         remove_surrogates(item[b'path']), extra))
         else:
@@ -734,17 +747,8 @@ class Archiver:
 
     def preprocess_args(self, args):
         deprecations = [
-            ('--hourly', '--keep-hourly', 'Warning: "--hourly" has been deprecated. Use "--keep-hourly" instead.'),
-            ('--daily', '--keep-daily', 'Warning: "--daily" has been deprecated. Use "--keep-daily" instead.'),
-            ('--weekly', '--keep-weekly', 'Warning: "--weekly" has been deprecated. Use "--keep-weekly" instead.'),
-            ('--monthly', '--keep-monthly', 'Warning: "--monthly" has been deprecated. Use "--keep-monthly" instead.'),
-            ('--yearly', '--keep-yearly', 'Warning: "--yearly" has been deprecated. Use "--keep-yearly" instead.'),
-            ('--do-not-cross-mountpoints', '--one-file-system',
-             'Warning:  "--do-no-cross-mountpoints" has been deprecated. Use "--one-file-system" instead.'),
+            #('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),
         ]
-        if args and args[0] == 'verify':
-            print('Warning: "borg verify" has been deprecated. Use "borg extract --dry-run" instead.')
-            args = ['extract', '--dry-run'] + args[1:]
         for i, arg in enumerate(args[:]):
             for old_name, new_name, warning in deprecations:
                 if arg.startswith(old_name):
@@ -789,8 +793,6 @@ class Archiver:
         This command initializes an empty repository. A repository is a filesystem
         directory containing the deduplicated data from zero or more archives.
         Encryption can be enabled at repository init time.
-        Please note that the 'passphrase' encryption mode is DEPRECATED (instead of it,
-        consider using 'repokey').
         """)
         subparser = subparsers.add_parser('init', parents=[common_parser],
                                           description=self.do_init.__doc__, epilog=init_epilog,
@@ -800,8 +802,8 @@ class Archiver:
                                type=location_validator(archive=False),
                                help='repository to create')
         subparser.add_argument('-e', '--encryption', dest='encryption',
-                               choices=('none', 'keyfile', 'repokey', 'passphrase'), default='none',
-                               help='select encryption key mode')
+                               choices=('none', 'keyfile', 'repokey'), default='repokey',
+                               help='select encryption key mode (default: "%(default)s")')
 
         check_epilog = textwrap.dedent("""
         The check command verifies the consistency of a repository and the corresponding archives.
@@ -877,6 +879,32 @@ class Archiver:
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
+        migrate_to_repokey_epilog = textwrap.dedent("""
+        This command migrates a repository from passphrase mode (not supported any
+        more) to repokey mode.
+
+        You will be first asked for the repository passphrase (to open it in passphrase
+        mode). This is the same passphrase as you used to use for this repo before 1.0.
+
+        It will then derive the different secrets from this passphrase.
+
+        Then you will be asked for a new passphrase (twice, for safety). This
+        passphrase will be used to protect the repokey (which contains these same
+        secrets in encrypted form). You may use the same passphrase as you used to
+        use, but you may also use a different one.
+
+        After migrating to repokey mode, you can change the passphrase at any time.
+        But please note: the secrets will always stay the same and they could always
+        be derived from your (old) passphrase-mode passphrase.
+        """)
+        subparser = subparsers.add_parser('migrate-to-repokey', parents=[common_parser],
+                                          description=self.do_migrate_to_repokey.__doc__,
+                                          epilog=migrate_to_repokey_epilog,
+                                          formatter_class=argparse.RawDescriptionHelpFormatter)
+        subparser.set_defaults(func=self.do_migrate_to_repokey)
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False))
+
         create_epilog = textwrap.dedent("""
         This command creates a backup archive containing all files found while recursively
         traversing all paths specified. The archive will consume almost no disk space for

borg/cache.py

@@ -3,13 +3,13 @@ from .remote import cache_if_remote
 from collections import namedtuple
 import os
 import stat
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 import shutil
 
 from .key import PlaintextKey
 from .logger import create_logger
 logger = create_logger()
-from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \
+from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \
     bigint_to_int, format_file_size, yes
 from .locking import UpgradableLock
 from .hashindex import ChunkIndex
@@ -54,8 +54,7 @@ class Cache:
                 msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" +
                        "\n" +
                        "Do you want to continue? [yN] ")
-                if not yes(msg, false_msg="Aborting.", default_notty=False,
-                           env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
+                if not yes(msg, false_msg="Aborting.", env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
                     raise self.CacheInitAbortedError()
             self.create()
         self.open(lock_wait=lock_wait)
@@ -64,8 +63,7 @@ class Cache:
             msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) +
                    "\n" +
                    "Do you want to continue? [yN] ")
-            if not yes(msg, false_msg="Aborting.", default_notty=False,
-                       env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
+            if not yes(msg, false_msg="Aborting.", env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
                 raise self.RepositoryAccessAborted()
 
         if sync and self.manifest.id != self.manifest_id:
@@ -136,7 +134,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                 raise Exception('%s has unexpected cache version %d (wanted: %d).' % (
                     config_path, cache_version, wanted_version))
         except configparser.NoSectionError as e:
-            raise Exception('%s does not look like a Borg cache.' % config_path)
+            raise Exception('%s does not look like a Borg cache.' % config_path) from None
         self.id = self.config.get('cache', 'repository')
         self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
         self.timestamp = self.config.get('cache', 'timestamp', fallback=None)
@@ -401,7 +399,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not entry:
             return None
         entry = msgpack.unpackb(entry)
-        if entry[2] == st.st_size and bigint_to_int(entry[3]) == st_mtime_ns(st) and entry[1] == st.st_ino:
+        if entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and entry[1] == st.st_ino:
             # reset entry age
             entry[0] = 0
             self.files[path_hash] = msgpack.packb(entry)
@@ -413,6 +411,6 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not (self.do_files and stat.S_ISREG(st.st_mode)):
             return
         # Entry: Age, inode, size, mtime, chunk ids
-        mtime_ns = st_mtime_ns(st)
+        mtime_ns = st.st_mtime_ns
         self.files[path_hash] = msgpack.packb((0, st.st_ino, st.st_size, int_to_bigint(mtime_ns), ids))
         self._newest_mtime = max(self._newest_mtime, mtime_ns)

borg/compress.pyx

@@ -110,7 +110,7 @@ cdef class LZ4(CompressorBase):
 
 class LZMA(CompressorBase):
     """
-    lzma compression / decompression (python 3.3+ stdlib)
+    lzma compression / decompression
     """
     ID = b'\x02\x00'
     name = 'lzma'

borg/crypto.pyx

@@ -1,7 +1,6 @@
 """A thin OpenSSL wrapper
 
-This could be replaced by PyCrypto or something similar when the performance
-of their PBKDF2 implementation is comparable to the OpenSSL version.
+This could be replaced by PyCrypto maybe?
 """
 from libc.stdlib cimport malloc, free
 
@@ -21,7 +20,6 @@ cdef extern from "openssl/evp.h":
         pass
     ctypedef struct ENGINE:
         pass
-    const EVP_MD *EVP_sha256()
     const EVP_CIPHER *EVP_aes_256_ctr()
     void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a)
     void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a)
@@ -37,10 +35,6 @@ cdef extern from "openssl/evp.h":
     int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
     int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
 
-    int PKCS5_PBKDF2_HMAC(const char *password, int passwordlen,
-                          const unsigned char *salt, int saltlen, int iter,
-                          const EVP_MD *digest,
-                          int keylen, unsigned char *out)
 
 import struct
 
@@ -59,35 +53,6 @@ def num_aes_blocks(int length):
     return (length + 15) // 16
 
 
-def pbkdf2_sha256(password, salt, iterations, size):
-    """Password based key derivation function 2 (RFC2898)
-    """
-    cdef unsigned char *key = <unsigned char *>malloc(size)
-    if not key:
-        raise MemoryError
-    try:
-        rv = PKCS5_PBKDF2_HMAC(password, len(password), salt, len(salt), iterations, EVP_sha256(), size, key)
-        if not rv:
-            raise Exception('PKCS5_PBKDF2_HMAC failed')
-        return key[:size]
-    finally:
-        free(key)
-
-
-def get_random_bytes(n):
-    """Return n cryptographically strong pseudo-random bytes
-    """
-    cdef unsigned char *buf = <unsigned char *>malloc(n)
-    if not buf:
-        raise MemoryError
-    try:
-        if RAND_bytes(buf, n) < 1:
-            raise Exception('RAND_bytes failed')
-        return buf[:n]
-    finally:
-        free(buf)
-
-
 cdef class AES:
     """A thin wrapper around the OpenSSL EVP cipher API
     """

borg/fuse.py

@@ -173,7 +173,7 @@ class FuseOperations(llfuse.Operations):
         try:
             return item.get(b'xattrs', {})[name]
         except KeyError:
-            raise llfuse.FUSEError(errno.ENODATA)
+            raise llfuse.FUSEError(errno.ENODATA) from None
 
     def _load_pending_archive(self, inode):
         # Check if this is an archive we need to load

borg/helpers.py

@@ -1,5 +1,4 @@
-from .support import argparse  # see support/__init__.py docstring, DEPRECATED - remove after requiring py 3.4
-
+import argparse
 import binascii
 from collections import namedtuple
 from functools import wraps
@@ -7,12 +6,7 @@ import grp
 import os
 import pwd
 import re
-try:
-    from shutil import get_terminal_size
-except ImportError:
-    def get_terminal_size(fallback=(80, 24)):
-        TerminalSize = namedtuple('TerminalSize', ['columns', 'lines'])
-        return TerminalSize(int(os.environ.get('COLUMNS', fallback[0])), int(os.environ.get('LINES', fallback[1])))
+from shutil import get_terminal_size
 import sys
 import platform
 import time
@@ -211,8 +205,7 @@ class Statistics:
                 msg += "{0:<{space}}".format(path, space=space)
             else:
                 msg = ' ' * columns
-            print(msg, file=stream or sys.stderr, end="\r")
-            (stream or sys.stderr).flush()
+            print(msg, file=stream or sys.stderr, end="\r", flush=True)
 
 
 def get_keys_dir():
@@ -473,35 +466,21 @@ def CompressionSpec(s):
     count = len(values)
     if count < 1:
         raise ValueError
-    compression = values[0]
-    try:
-        compression = int(compression)
-        if count > 1:
-            raise ValueError
-        # DEPRECATED: it is just --compression N
-        if 0 <= compression <= 9:
-            print('Warning: --compression %d is deprecated, please use --compression zlib,%d.' % (compression, compression))
-            if compression == 0:
-                print('Hint: instead of --compression zlib,0 you could also use --compression none for better performance.')
-                print('Hint: archives generated using --compression none are not compatible with borg < 0.25.0.')
-            return dict(name='zlib', level=compression)
-        raise ValueError
-    except ValueError:
-        # --compression algo[,...]
-        name = compression
-        if name in ('none', 'lz4', ):
-            return dict(name=name)
-        if name in ('zlib', 'lzma', ):
-            if count < 2:
-                level = 6  # default compression level in py stdlib
-            elif count == 2:
-                level = int(values[1])
-                if not 0 <= level <= 9:
-                    raise ValueError
-            else:
+    # --compression algo[,level]
+    name = values[0]
+    if name in ('none', 'lz4', ):
+        return dict(name=name)
+    if name in ('zlib', 'lzma', ):
+        if count < 2:
+            level = 6  # default compression level in py stdlib
+        elif count == 2:
+            level = int(values[1])
+            if not 0 <= level <= 9:
                 raise ValueError
-            return dict(name=name, level=level)
-        raise ValueError
+        else:
+            raise ValueError
+        return dict(name=name, level=level)
+    raise ValueError
 
 
 def dir_is_cachedir(path):
@@ -565,15 +544,6 @@ def format_timedelta(td):
     return txt
 
 
-def format_file_mode(mod):
-    """Format file mode bits for list output
-    """
-    def x(v):
-        return ''.join(v & m and s or '-'
-                       for m, s in ((4, 'r'), (2, 'w'), (1, 'x')))
-    return '%s%s%s' % (x(mod // 64), x(mod // 8), x(mod))
-
-
 def format_file_size(v, precision=2):
     """Format file size into a human friendly format
     """
@@ -779,7 +749,7 @@ def location_validator(archive=None):
         try:
             loc = Location(text)
         except ValueError:
-            raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text)
+            raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) from None
         if archive is True and not loc.archive:
             raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
         elif archive is False and loc.archive:
@@ -836,35 +806,6 @@ class StableDict(dict):
         return sorted(super().items())
 
 
-if sys.version < '3.3':
-    # st_xtime_ns attributes only available in 3.3+
-    def st_atime_ns(st):
-        return int(st.st_atime * 1e9)
-
-    def st_ctime_ns(st):
-        return int(st.st_ctime * 1e9)
-
-    def st_mtime_ns(st):
-        return int(st.st_mtime * 1e9)
-
-    # unhexlify in < 3.3 incorrectly only accepts bytes input
-    def unhexlify(data):
-        if isinstance(data, str):
-            data = data.encode('ascii')
-        return binascii.unhexlify(data)
-else:
-    def st_atime_ns(st):
-        return st.st_atime_ns
-
-    def st_ctime_ns(st):
-        return st.st_ctime_ns
-
-    def st_mtime_ns(st):
-        return st.st_mtime_ns
-
-    unhexlify = binascii.unhexlify
-
-
 def bigint_to_int(mtime):
     """Convert bytearray to int
     """
@@ -887,71 +828,69 @@ def is_slow_msgpack():
     return msgpack.Packer is msgpack.fallback.Packer
 
 
-def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None,
-        default=False, default_notty=None, default_eof=None,
-        falsish=('No', 'no', 'N', 'n'), truish=('Yes', 'yes', 'Y', 'y'),
-        env_var_override=None, ifile=None, ofile=None, input=input):
+FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', )
+TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', )
+DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', )
+
+def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
+        retry_msg=None, invalid_msg=None, env_msg=None,
+        falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH,
+        default=False, retry=True, env_var_override=None, ofile=None, input=input):
     """
     Output <msg> (usually a question) and let user input an answer.
-    Qualifies the answer according to falsish and truish as True or False.
+    Qualifies the answer according to falsish, truish and defaultish as True, False or <default>.
     If it didn't qualify and retry_msg is None (no retries wanted),
     return the default [which defaults to False]. Otherwise let user retry
     answering until answer is qualified.
 
-    If env_var_override is given and it is non-empty, counts as truish answer
-    and won't ask user for an answer.
-    If we don't have a tty as input and default_notty is not None, return its value.
-    Otherwise read input from non-tty and proceed as normal.
-    If EOF is received instead an input, return default_eof [or default, if not given].
+    If env_var_override is given and this var is present in the environment, do not ask
+    the user, but just use the env var contents as answer as if it was typed in.
+    Otherwise read input from stdin and proceed as normal.
+    If EOF is received instead an input or an invalid input without retry possibility,
+    return default.
 
     :param msg: introducing message to output on ofile, no \n is added [None]
     :param retry_msg: retry message to output on ofile, no \n is added [None]
-           (also enforces retries instead of returning default)
     :param false_msg: message to output before returning False [None]
     :param true_msg: message to output before returning True [None]
-    :param default: default return value (empty answer is given) [False]
-    :param default_notty: if not None, return its value if no tty is connected [None]
-    :param default_eof: return value if EOF was read as answer [same as default]
+    :param default_msg: message to output before returning a <default> [None]
+    :param invalid_msg: message to output after a invalid answer was given [None]
+    :param env_msg: message to output when using input from env_var_override [None],
+           needs to have 2 placeholders for answer and env var name, e.g.: "{} (from {})"
     :param falsish: sequence of answers qualifying as False
     :param truish: sequence of answers qualifying as True
+    :param defaultish: sequence of answers qualifying as <default>
+    :param default: default return value (defaultish answer was given or no-answer condition) [False]
+    :param retry: if True and input is incorrect, retry. Otherwise return default. [True]
     :param env_var_override: environment variable name [None]
-    :param ifile: input stream [sys.stdin] (only for testing!)
     :param ofile: output stream [sys.stderr]
     :param input: input function [input from builtins]
     :return: boolean answer value, True or False
     """
-    # note: we do not assign sys.stdin/stderr as defaults above, so they are
+    # note: we do not assign sys.stderr as default above, so it is
     # really evaluated NOW,  not at function definition time.
-    if ifile is None:
-        ifile = sys.stdin
     if ofile is None:
         ofile = sys.stderr
     if default not in (True, False):
         raise ValueError("invalid default value, must be True or False")
-    if default_notty not in (None, True, False):
-        raise ValueError("invalid default_notty value, must be None, True or False")
-    if default_eof not in (None, True, False):
-        raise ValueError("invalid default_eof value, must be None, True or False")
     if msg:
-        print(msg, file=ofile, end='')
-        ofile.flush()
-    if env_var_override:
-        value = os.environ.get(env_var_override)
-        # currently, any non-empty value counts as truish
-        # TODO: change this so one can give y/n there?
-        if value:
-            value = bool(value)
-            value_str = truish[0] if value else falsish[0]
-            print("{} (from {})".format(value_str, env_var_override), file=ofile)
-            return value
-    if default_notty is not None and not ifile.isatty():
-        # looks like ifile is not a terminal (but e.g. a pipe)
-        return default_notty
+        print(msg, file=ofile, end='', flush=True)
     while True:
-        try:
-            answer = input()  # XXX how can we use ifile?
-        except EOFError:
-            return default_eof if default_eof is not None else default
+        answer = None
+        if env_var_override:
+            answer = os.environ.get(env_var_override)
+            if answer is not None and env_msg:
+                print(env_msg.format(answer, env_var_override), file=ofile)
+        if answer is None:
+            try:
+                answer = input()
+            except EOFError:
+                # avoid defaultish[0], defaultish could be empty
+                answer = truish[0] if default else falsish[0]
+        if answer in defaultish:
+            if default_msg:
+                print(default_msg, file=ofile)
+            return default
         if answer in truish:
             if true_msg:
                 print(true_msg, file=ofile)
@@ -960,12 +899,15 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None,
             if false_msg:
                 print(false_msg, file=ofile)
             return False
-        if retry_msg is None:
-            # no retries wanted, we just return the default
+        # if we get here, the answer was invalid
+        if invalid_msg:
+            print(invalid_msg, file=ofile)
+        if not retry:
             return default
         if retry_msg:
-            print(retry_msg, file=ofile, end='')
-            ofile.flush()
+            print(retry_msg, file=ofile, end='', flush=True)
+        # in case we used an environment variable and it gave an invalid answer, do not use it again:
+        env_var_override = None
 
 
 class ProgressIndicatorPercent:
@@ -1003,8 +945,7 @@ class ProgressIndicatorPercent:
             return self.output(pct)
 
     def output(self, percent):
-        print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n')  # python 3.3 gives us flush=True
-        self.file.flush()
+        print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n', flush=True)
 
     def finish(self):
         if self.same_line:
@@ -1038,8 +979,7 @@ class ProgressIndicatorEndless:
             return self.output(self.triggered)
 
     def output(self, triggered):
-        print('.', end='', file=self.file)  # python 3.3 gives us flush=True
-        self.file.flush()
+        print('.', end='', file=self.file, flush=True)
 
     def finish(self):
         print(file=self.file)

borg/key.py

@@ -4,14 +4,14 @@ import getpass
 import os
 import sys
 import textwrap
-import hmac
-from hashlib import sha256
+from hmac import HMAC, compare_digest
+from hashlib import sha256, pbkdf2_hmac
 
 from .helpers import IntegrityError, get_keys_dir, Error
 from .logger import create_logger
 logger = create_logger()
 
-from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
+from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
 from .compress import Compressor, COMPR_BUFFER
 import msgpack
 
@@ -30,20 +30,11 @@ class RepoKeyNotFoundError(Error):
     """No key entry found in the config of repository {}."""
 
 
-class HMAC(hmac.HMAC):
-    """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews
-    """
-    def update(self, msg):
-        self.inner.update(msg)
-
-
 def key_creator(repository, args):
     if args.encryption == 'keyfile':
         return KeyfileKey.create(repository, args)
     elif args.encryption == 'repokey':
         return RepoKey.create(repository, args)
-    elif args.encryption == 'passphrase':  # deprecated, kill in 1.x
-        return PassphraseKey.create(repository, args)
     else:
         return PlaintextKey.create(repository, args)
 
@@ -54,8 +45,10 @@ def key_factory(repository, manifest_data):
         return KeyfileKey.detect(repository, manifest_data)
     elif key_type == RepoKey.TYPE:
         return RepoKey.detect(repository, manifest_data)
-    elif key_type == PassphraseKey.TYPE:  # deprecated, kill in 1.x
-        return PassphraseKey.detect(repository, manifest_data)
+    elif key_type == PassphraseKey.TYPE:
+        # we just dispatch to repokey mode and assume the passphrase was migrated to a repokey.
+        # see also comment in PassphraseKey class.
+        return RepoKey.detect(repository, manifest_data)
     elif key_type == PlaintextKey.TYPE:
         return PlaintextKey.detect(repository, manifest_data)
     else:
@@ -139,19 +132,25 @@ class AESKeyBase(KeyBase):
         return b''.join((self.TYPE_STR, hmac, data))
 
     def decrypt(self, id, data):
-        if data[0] != self.TYPE:
+        if not (data[0] == self.TYPE or \
+            data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
             raise IntegrityError('Invalid encryption envelope')
-        hmac = memoryview(data)[1:33]
-        if memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) != hmac:
+        hmac_given = memoryview(data)[1:33]
+        hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest())
+        if not compare_digest(hmac_computed, hmac_given):
             raise IntegrityError('Encryption envelope checksum mismatch')
         self.dec_cipher.reset(iv=PREFIX + data[33:41])
         data = self.compressor.decompress(self.dec_cipher.decrypt(data[41:]))
-        if id and HMAC(self.id_key, data, sha256).digest() != id:
-            raise IntegrityError('Chunk id verification failed')
+        if id:
+            hmac_given = id
+            hmac_computed = HMAC(self.id_key, data, sha256).digest()
+            if not compare_digest(hmac_computed, hmac_given):
+                raise IntegrityError('Chunk id verification failed')
         return data
 
     def extract_nonce(self, payload):
-        if payload[0] != self.TYPE:
+        if not (payload[0] == self.TYPE or \
+            payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
             raise IntegrityError('Invalid encryption envelope')
         nonce = bytes_to_long(payload[33:41])
         return nonce
@@ -202,22 +201,25 @@ class Passphrase(str):
         return '<Passphrase "***hidden***">'
 
     def kdf(self, salt, iterations, length):
-        return pbkdf2_sha256(self.encode('utf-8'), salt, iterations, length)
+        return pbkdf2_hmac('sha256', self.encode('utf-8'), salt, iterations, length)
 
 
 class PassphraseKey(AESKeyBase):
-    # This mode is DEPRECATED and will be killed at 1.0 release.
-    # With this mode:
+    # This mode was killed in borg 1.0, see: https://github.com/borgbackup/borg/issues/97
+    # Reasons:
     # - you can never ever change your passphrase for existing repos.
     # - you can never ever use a different iterations count for existing repos.
+    # "Killed" means:
+    # - there is no automatic dispatch to this class via type byte
+    # - --encryption=passphrase is an invalid argument now
+    # This class is kept for a while to support migration from passphrase to repokey mode.
     TYPE = 0x01
     iterations = 100000  # must not be changed ever!
 
     @classmethod
     def create(cls, repository, args):
         key = cls(repository)
-        logger.warning('WARNING: "passphrase" mode is deprecated and will be removed in 1.0.')
-        logger.warning('If you want something similar (but with less issues), use "repokey" mode.')
+        logger.warning('WARNING: "passphrase" mode is unsupported since borg 1.0.')
         passphrase = Passphrase.new(allow_empty=False)
         key.init(repository, passphrase)
         return key
@@ -270,7 +272,7 @@ class KeyfileKeyBase(AESKeyBase):
         raise NotImplementedError
 
     def _load(self, key_data, passphrase):
-        cdata = a2b_base64(key_data.encode('ascii'))  # .encode needed for Python 3.[0-2]
+        cdata = a2b_base64(key_data)
         data = self.decrypt_key_file(cdata, passphrase)
         if data:
             key = msgpack.unpackb(data)
@@ -294,7 +296,7 @@ class KeyfileKeyBase(AESKeyBase):
             return data
 
     def encrypt_key_file(self, data, passphrase):
-        salt = get_random_bytes(32)
+        salt = os.urandom(32)
         iterations = 100000
         key = passphrase.kdf(salt, iterations, 32)
         hash = HMAC(key, data, sha256).digest()
@@ -332,7 +334,7 @@ class KeyfileKeyBase(AESKeyBase):
         passphrase = Passphrase.new(allow_empty=True)
         key = cls(repository)
         key.repository_id = repository.id
-        key.init_from_random_data(get_random_bytes(100))
+        key.init_from_random_data(os.urandom(100))
         key.init_ciphers()
         target = key.get_new_target(args)
         key.save(target, passphrase)
@@ -397,7 +399,7 @@ class RepoKey(KeyfileKeyBase):
             self.repository.load_key()
             return loc
         except configparser.NoOptionError:
-            raise RepoKeyNotFoundError(loc)
+            raise RepoKeyNotFoundError(loc) from None
 
     def get_new_target(self, args):
         return self.repository

borg/locking.py

@@ -132,14 +132,13 @@ class ExclusiveLock:
         while True:
             try:
                 os.mkdir(self.path)
+            except FileExistsError:  # already locked
+                if self.by_me():
+                    return self
+                if timer.timed_out_or_sleep():
+                    raise LockTimeout(self.path)
             except OSError as err:
-                if err.errno == errno.EEXIST:  # already locked
-                    if self.by_me():
-                        return self
-                    if timer.timed_out_or_sleep():
-                        raise LockTimeout(self.path)
-                else:
-                    raise LockFailed(self.path, str(err))
+                raise LockFailed(self.path, str(err)) from None
             else:
                 with open(self.unique_name, "wb"):
                     pass
@@ -181,12 +180,8 @@ class LockRoster:
         try:
             with open(self.path) as f:
                 data = json.load(f)
-        except IOError as err:
-            if err.errno != errno.ENOENT:
-                raise
-            data = {}
-        except ValueError:
-            # corrupt/empty roster file?
+        except (FileNotFoundError, ValueError):
+            # no or corrupt/empty roster file?
             data = {}
         return data
 
@@ -197,9 +192,8 @@ class LockRoster:
     def remove(self):
         try:
             os.unlink(self.path)
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                raise
+        except FileNotFoundError:
+            pass
 
     def get(self, key):
         roster = self.load()

borg/remote.py

@@ -129,7 +129,7 @@ class RemoteRepository:
             self.name = name
 
     def __init__(self, location, create=False, lock_wait=None, lock=True, args=None):
-        self.location = location
+        self.location = self._location = location
         self.preload_ids = []
         self.msgid = 0
         self.to_send = b''
@@ -159,7 +159,7 @@ class RemoteRepository:
         try:
             version = self.call('negotiate', RPC_PROTOCOL_VERSION)
         except ConnectionClosed:
-            raise ConnectionClosedWithHint('Is borg working on the server?')
+            raise ConnectionClosedWithHint('Is borg working on the server?') from None
         if version != RPC_PROTOCOL_VERSION:
             raise Exception('Server insisted on using unsupported protocol version %d' % version)
         self.id = self.call('open', location.path, create, lock_wait, lock)
@@ -268,7 +268,7 @@ class RemoteRepository:
                     if not data:
                         raise ConnectionClosed()
                     data = data.decode('utf-8')
-                    for line in data.splitlines(True):  # keepends=True for py3.3+
+                    for line in data.splitlines(keepends=True):
                         if line.startswith('$LOG '):
                             _, level, msg = line.split(' ', 2)
                             level = getattr(logging, level, logging.CRITICAL)  # str -> int

borg/repository.py

@@ -1,5 +1,5 @@
 from configparser import ConfigParser
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 from itertools import islice
 import errno
 import logging
@@ -11,7 +11,7 @@ import struct
 from zlib import crc32
 
 import msgpack
-from .helpers import Error, ErrorWithTraceback, IntegrityError, unhexlify, ProgressIndicatorPercent
+from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent
 from .hashindex import NSIndex
 from .locking import UpgradableLock, LockError, LockErrorT
 from .lrucache import LRUCache
@@ -54,6 +54,7 @@ class Repository:
 
     def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True):
         self.path = os.path.abspath(path)
+        self._location = Location('file://%s' % self.path)
         self.io = None
         self.lock = None
         self.index = None
@@ -417,7 +418,7 @@ class Repository:
             segment, offset = self.index[id_]
             return self.io.read(segment, offset, id_)
         except KeyError:
-            raise self.ObjectNotFound(id_, self.path)
+            raise self.ObjectNotFound(id_, self.path) from None
 
     def get_many(self, ids, is_preloaded=False):
         for id_ in ids:
@@ -446,7 +447,7 @@ class Repository:
         try:
             segment, offset = self.index.pop(id)
         except KeyError:
-            raise self.ObjectNotFound(id, self.path)
+            raise self.ObjectNotFound(id, self.path) from None
         self.segments[segment] -= 1
         self.compact.add(segment)
         segment = self.io.write_delete(id)
@@ -567,7 +568,7 @@ class LoggedIO:
             del self.fds[segment]
         try:
             os.unlink(self.segment_filename(segment))
-        except OSError:
+        except FileNotFoundError:
             pass
 
     def segment_exists(self, segment):
@@ -628,7 +629,7 @@ class LoggedIO:
             hdr_tuple = fmt.unpack(header)
         except struct.error as err:
             raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format(
-                segment, offset, err))
+                segment, offset, err)) from None
         if fmt is self.put_header_fmt:
             crc, size, tag, key = hdr_tuple
         elif fmt is self.header_fmt:
@@ -685,7 +686,7 @@ class LoggedIO:
             self.offset = 0
             self._write_fd.flush()
             os.fsync(self._write_fd.fileno())
-            if hasattr(os, 'posix_fadvise'):  # python >= 3.3, only on UNIX
+            if hasattr(os, 'posix_fadvise'):  # only on UNIX
                 # tell the OS that it does not need to cache what we just wrote,
                 # avoids spoiling the cache for the OS and other processes.
                 os.posix_fadvise(self._write_fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)

borg/support/__init__.py

@@ -1,16 +0,0 @@
-"""
-3rd party stuff that needed fixing
-
-Note: linux package maintainers feel free to remove any of these hacks
-      IF your python version is not affected.
-
-argparse is broken with default args (double conversion):
-affects: 3.2.0 <= python < 3.2.4
-affects: 3.3.0 <= python < 3.3.1
-
-as we still support 3.2 and 3.3 there is no other way than to bundle
-a fixed version (I just took argparse.py from 3.2.6) and import it from
-here (see import in archiver.py).
-DEPRECATED - remove support.argparse after requiring python 3.4.
-"""
-

borg/support/argparse.py

@@ -1,2383 +0,0 @@
-# Author: Steven J. Bethard <steven.bethard@gmail.com>.
-
-"""Command-line parsing library
-
-This module is an optparse-inspired command-line parsing library that:
-
-    - handles both optional and positional arguments
-    - produces highly informative usage messages
-    - supports parsers that dispatch to sub-parsers
-
-The following is a simple usage example that sums integers from the
-command-line and writes the result to a file::
-
-    parser = argparse.ArgumentParser(
-        description='sum the integers at the command line')
-    parser.add_argument(
-        'integers', metavar='int', nargs='+', type=int,
-        help='an integer to be summed')
-    parser.add_argument(
-        '--log', default=sys.stdout, type=argparse.FileType('w'),
-        help='the file where the sum should be written')
-    args = parser.parse_args()
-    args.log.write('%s' % sum(args.integers))
-    args.log.close()
-
-The module contains the following public classes:
-
-    - ArgumentParser -- The main entry point for command-line parsing. As the
-        example above shows, the add_argument() method is used to populate
-        the parser with actions for optional and positional arguments. Then
-        the parse_args() method is invoked to convert the args at the
-        command-line into an object with attributes.
-
-    - ArgumentError -- The exception raised by ArgumentParser objects when
-        there are errors with the parser's actions. Errors raised while
-        parsing the command-line are caught by ArgumentParser and emitted
-        as command-line messages.
-
-    - FileType -- A factory for defining types of files to be created. As the
-        example above shows, instances of FileType are typically passed as
-        the type= argument of add_argument() calls.
-
-    - Action -- The base class for parser actions. Typically actions are
-        selected by passing strings like 'store_true' or 'append_const' to
-        the action= argument of add_argument(). However, for greater
-        customization of ArgumentParser actions, subclasses of Action may
-        be defined and passed as the action= argument.
-
-    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
-        ArgumentDefaultsHelpFormatter -- Formatter classes which
-        may be passed as the formatter_class= argument to the
-        ArgumentParser constructor. HelpFormatter is the default,
-        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
-        not to change the formatting for help text, and
-        ArgumentDefaultsHelpFormatter adds information about argument defaults
-        to the help.
-
-All other classes in this module are considered implementation details.
-(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
-considered public as object names -- the API of the formatter objects is
-still considered an implementation detail.)
-"""
-
-__version__ = '1.1'
-__all__ = [
-    'ArgumentParser',
-    'ArgumentError',
-    'ArgumentTypeError',
-    'FileType',
-    'HelpFormatter',
-    'ArgumentDefaultsHelpFormatter',
-    'RawDescriptionHelpFormatter',
-    'RawTextHelpFormatter',
-    'Namespace',
-    'Action',
-    'ONE_OR_MORE',
-    'OPTIONAL',
-    'PARSER',
-    'REMAINDER',
-    'SUPPRESS',
-    'ZERO_OR_MORE',
-]
-
-
-import collections as _collections
-import copy as _copy
-import os as _os
-import re as _re
-import sys as _sys
-import textwrap as _textwrap
-
-try:
-    from gettext import gettext, ngettext
-except ImportError:
-    def gettext(message):
-        return message
-    def ngettext(msg1, msg2, n):
-        return msg1 if n == 1 else msg2
-_ = gettext
-
-
-SUPPRESS = '==SUPPRESS=='
-
-OPTIONAL = '?'
-ZERO_OR_MORE = '*'
-ONE_OR_MORE = '+'
-PARSER = 'A...'
-REMAINDER = '...'
-_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
-
-# =============================
-# Utility functions and classes
-# =============================
-
-class _AttributeHolder(object):
-    """Abstract base class that provides __repr__.
-
-    The __repr__ method returns a string in the format::
-        ClassName(attr=name, attr=name, ...)
-    The attributes are determined either by a class-level attribute,
-    '_kwarg_names', or by inspecting the instance __dict__.
-    """
-
-    def __repr__(self):
-        type_name = type(self).__name__
-        arg_strings = []
-        for arg in self._get_args():
-            arg_strings.append(repr(arg))
-        for name, value in self._get_kwargs():
-            arg_strings.append('%s=%r' % (name, value))
-        return '%s(%s)' % (type_name, ', '.join(arg_strings))
-
-    def _get_kwargs(self):
-        return sorted(self.__dict__.items())
-
-    def _get_args(self):
-        return []
-
-
-def _ensure_value(namespace, name, value):
-    if getattr(namespace, name, None) is None:
-        setattr(namespace, name, value)
-    return getattr(namespace, name)
-
-
-# ===============
-# Formatting Help
-# ===============
-
-class HelpFormatter(object):
-    """Formatter for generating usage messages and argument help strings.
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def __init__(self,
-                 prog,
-                 indent_increment=2,
-                 max_help_position=24,
-                 width=None):
-
-        # default setting for width
-        if width is None:
-            try:
-                width = int(_os.environ['COLUMNS'])
-            except (KeyError, ValueError):
-                width = 80
-            width -= 2
-
-        self._prog = prog
-        self._indent_increment = indent_increment
-        self._max_help_position = max_help_position
-        self._width = width
-
-        self._current_indent = 0
-        self._level = 0
-        self._action_max_length = 0
-
-        self._root_section = self._Section(self, None)
-        self._current_section = self._root_section
-
-        self._whitespace_matcher = _re.compile(r'\s+')
-        self._long_break_matcher = _re.compile(r'\n\n\n+')
-
-    # ===============================
-    # Section and indentation methods
-    # ===============================
-    def _indent(self):
-        self._current_indent += self._indent_increment
-        self._level += 1
-
-    def _dedent(self):
-        self._current_indent -= self._indent_increment
-        assert self._current_indent >= 0, 'Indent decreased below 0.'
-        self._level -= 1
-
-    class _Section(object):
-
-        def __init__(self, formatter, parent, heading=None):
-            self.formatter = formatter
-            self.parent = parent
-            self.heading = heading
-            self.items = []
-
-        def format_help(self):
-            # format the indented section
-            if self.parent is not None:
-                self.formatter._indent()
-            join = self.formatter._join_parts
-            for func, args in self.items:
-                func(*args)
-            item_help = join([func(*args) for func, args in self.items])
-            if self.parent is not None:
-                self.formatter._dedent()
-
-            # return nothing if the section was empty
-            if not item_help:
-                return ''
-
-            # add the heading if the section was non-empty
-            if self.heading is not SUPPRESS and self.heading is not None:
-                current_indent = self.formatter._current_indent
-                heading = '%*s%s:\n' % (current_indent, '', self.heading)
-            else:
-                heading = ''
-
-            # join the section-initial newline, the heading and the help
-            return join(['\n', heading, item_help, '\n'])
-
-    def _add_item(self, func, args):
-        self._current_section.items.append((func, args))
-
-    # ========================
-    # Message building methods
-    # ========================
-    def start_section(self, heading):
-        self._indent()
-        section = self._Section(self, self._current_section, heading)
-        self._add_item(section.format_help, [])
-        self._current_section = section
-
-    def end_section(self):
-        self._current_section = self._current_section.parent
-        self._dedent()
-
-    def add_text(self, text):
-        if text is not SUPPRESS and text is not None:
-            self._add_item(self._format_text, [text])
-
-    def add_usage(self, usage, actions, groups, prefix=None):
-        if usage is not SUPPRESS:
-            args = usage, actions, groups, prefix
-            self._add_item(self._format_usage, args)
-
-    def add_argument(self, action):
-        if action.help is not SUPPRESS:
-
-            # find all invocations
-            get_invocation = self._format_action_invocation
-            invocations = [get_invocation(action)]
-            for subaction in self._iter_indented_subactions(action):
-                invocations.append(get_invocation(subaction))
-
-            # update the maximum item length
-            invocation_length = max([len(s) for s in invocations])
-            action_length = invocation_length + self._current_indent
-            self._action_max_length = max(self._action_max_length,
-                                          action_length)
-
-            # add the item to the list
-            self._add_item(self._format_action, [action])
-
-    def add_arguments(self, actions):
-        for action in actions:
-            self.add_argument(action)
-
-    # =======================
-    # Help-formatting methods
-    # =======================
-    def format_help(self):
-        help = self._root_section.format_help()
-        if help:
-            help = self._long_break_matcher.sub('\n\n', help)
-            help = help.strip('\n') + '\n'
-        return help
-
-    def _join_parts(self, part_strings):
-        return ''.join([part
-                        for part in part_strings
-                        if part and part is not SUPPRESS])
-
-    def _format_usage(self, usage, actions, groups, prefix):
-        if prefix is None:
-            prefix = _('usage: ')
-
-        # if usage is specified, use that
-        if usage is not None:
-            usage = usage % dict(prog=self._prog)
-
-        # if no optionals or positionals are available, usage is just prog
-        elif usage is None and not actions:
-            usage = '%(prog)s' % dict(prog=self._prog)
-
-        # if optionals and positionals are available, calculate usage
-        elif usage is None:
-            prog = '%(prog)s' % dict(prog=self._prog)
-
-            # split optionals from positionals
-            optionals = []
-            positionals = []
-            for action in actions:
-                if action.option_strings:
-                    optionals.append(action)
-                else:
-                    positionals.append(action)
-
-            # build full usage string
-            format = self._format_actions_usage
-            action_usage = format(optionals + positionals, groups)
-            usage = ' '.join([s for s in [prog, action_usage] if s])
-
-            # wrap the usage parts if it's too long
-            text_width = self._width - self._current_indent
-            if len(prefix) + len(usage) > text_width:
-
-                # break usage into wrappable parts
-                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
-                opt_usage = format(optionals, groups)
-                pos_usage = format(positionals, groups)
-                opt_parts = _re.findall(part_regexp, opt_usage)
-                pos_parts = _re.findall(part_regexp, pos_usage)
-                assert ' '.join(opt_parts) == opt_usage
-                assert ' '.join(pos_parts) == pos_usage
-
-                # helper for wrapping lines
-                def get_lines(parts, indent, prefix=None):
-                    lines = []
-                    line = []
-                    if prefix is not None:
-                        line_len = len(prefix) - 1
-                    else:
-                        line_len = len(indent) - 1
-                    for part in parts:
-                        if line_len + 1 + len(part) > text_width:
-                            lines.append(indent + ' '.join(line))
-                            line = []
-                            line_len = len(indent) - 1
-                        line.append(part)
-                        line_len += len(part) + 1
-                    if line:
-                        lines.append(indent + ' '.join(line))
-                    if prefix is not None:
-                        lines[0] = lines[0][len(indent):]
-                    return lines
-
-                # if prog is short, follow it with optionals or positionals
-                if len(prefix) + len(prog) <= 0.75 * text_width:
-                    indent = ' ' * (len(prefix) + len(prog) + 1)
-                    if opt_parts:
-                        lines = get_lines([prog] + opt_parts, indent, prefix)
-                        lines.extend(get_lines(pos_parts, indent))
-                    elif pos_parts:
-                        lines = get_lines([prog] + pos_parts, indent, prefix)
-                    else:
-                        lines = [prog]
-
-                # if prog is long, put it on its own line
-                else:
-                    indent = ' ' * len(prefix)
-                    parts = opt_parts + pos_parts
-                    lines = get_lines(parts, indent)
-                    if len(lines) > 1:
-                        lines = []
-                        lines.extend(get_lines(opt_parts, indent))
-                        lines.extend(get_lines(pos_parts, indent))
-                    lines = [prog] + lines
-
-                # join lines into usage
-                usage = '\n'.join(lines)
-
-        # prefix with 'usage:'
-        return '%s%s\n\n' % (prefix, usage)
-
-    def _format_actions_usage(self, actions, groups):
-        # find group indices and identify actions in groups
-        group_actions = set()
-        inserts = {}
-        for group in groups:
-            try:
-                start = actions.index(group._group_actions[0])
-            except ValueError:
-                continue
-            else:
-                end = start + len(group._group_actions)
-                if actions[start:end] == group._group_actions:
-                    for action in group._group_actions:
-                        group_actions.add(action)
-                    if not group.required:
-                        if start in inserts:
-                            inserts[start] += ' ['
-                        else:
-                            inserts[start] = '['
-                        inserts[end] = ']'
-                    else:
-                        if start in inserts:
-                            inserts[start] += ' ('
-                        else:
-                            inserts[start] = '('
-                        inserts[end] = ')'
-                    for i in range(start + 1, end):
-                        inserts[i] = '|'
-
-        # collect all actions format strings
-        parts = []
-        for i, action in enumerate(actions):
-
-            # suppressed arguments are marked with None
-            # remove | separators for suppressed arguments
-            if action.help is SUPPRESS:
-                parts.append(None)
-                if inserts.get(i) == '|':
-                    inserts.pop(i)
-                elif inserts.get(i + 1) == '|':
-                    inserts.pop(i + 1)
-
-            # produce all arg strings
-            elif not action.option_strings:
-                part = self._format_args(action, action.dest)
-
-                # if it's in a group, strip the outer []
-                if action in group_actions:
-                    if part[0] == '[' and part[-1] == ']':
-                        part = part[1:-1]
-
-                # add the action string to the list
-                parts.append(part)
-
-            # produce the first way to invoke the option in brackets
-            else:
-                option_string = action.option_strings[0]
-
-                # if the Optional doesn't take a value, format is:
-                #    -s or --long
-                if action.nargs == 0:
-                    part = '%s' % option_string
-
-                # if the Optional takes a value, format is:
-                #    -s ARGS or --long ARGS
-                else:
-                    default = action.dest.upper()
-                    args_string = self._format_args(action, default)
-                    part = '%s %s' % (option_string, args_string)
-
-                # make it look optional if it's not required or in a group
-                if not action.required and action not in group_actions:
-                    part = '[%s]' % part
-
-                # add the action string to the list
-                parts.append(part)
-
-        # insert things at the necessary indices
-        for i in sorted(inserts, reverse=True):
-            parts[i:i] = [inserts[i]]
-
-        # join all the action items with spaces
-        text = ' '.join([item for item in parts if item is not None])
-
-        # clean up separators for mutually exclusive groups
-        open = r'[\[(]'
-        close = r'[\])]'
-        text = _re.sub(r'(%s) ' % open, r'\1', text)
-        text = _re.sub(r' (%s)' % close, r'\1', text)
-        text = _re.sub(r'%s *%s' % (open, close), r'', text)
-        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
-        text = text.strip()
-
-        # return the text
-        return text
-
-    def _format_text(self, text):
-        if '%(prog)' in text:
-            text = text % dict(prog=self._prog)
-        text_width = self._width - self._current_indent
-        indent = ' ' * self._current_indent
-        return self._fill_text(text, text_width, indent) + '\n\n'
-
-    def _format_action(self, action):
-        # determine the required width and the entry label
-        help_position = min(self._action_max_length + 2,
-                            self._max_help_position)
-        help_width = self._width - help_position
-        action_width = help_position - self._current_indent - 2
-        action_header = self._format_action_invocation(action)
-
-        # ho nelp; start on same line and add a final newline
-        if not action.help:
-            tup = self._current_indent, '', action_header
-            action_header = '%*s%s\n' % tup
-
-        # short action name; start on the same line and pad two spaces
-        elif len(action_header) <= action_width:
-            tup = self._current_indent, '', action_width, action_header
-            action_header = '%*s%-*s  ' % tup
-            indent_first = 0
-
-        # long action name; start on the next line
-        else:
-            tup = self._current_indent, '', action_header
-            action_header = '%*s%s\n' % tup
-            indent_first = help_position
-
-        # collect the pieces of the action help
-        parts = [action_header]
-
-        # if there was help for the action, add lines of help text
-        if action.help:
-            help_text = self._expand_help(action)
-            help_lines = self._split_lines(help_text, help_width)
-            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
-            for line in help_lines[1:]:
-                parts.append('%*s%s\n' % (help_position, '', line))
-
-        # or add a newline if the description doesn't end with one
-        elif not action_header.endswith('\n'):
-            parts.append('\n')
-
-        # if there are any sub-actions, add their help as well
-        for subaction in self._iter_indented_subactions(action):
-            parts.append(self._format_action(subaction))
-
-        # return a single string
-        return self._join_parts(parts)
-
-    def _format_action_invocation(self, action):
-        if not action.option_strings:
-            metavar, = self._metavar_formatter(action, action.dest)(1)
-            return metavar
-
-        else:
-            parts = []
-
-            # if the Optional doesn't take a value, format is:
-            #    -s, --long
-            if action.nargs == 0:
-                parts.extend(action.option_strings)
-
-            # if the Optional takes a value, format is:
-            #    -s ARGS, --long ARGS
-            else:
-                default = action.dest.upper()
-                args_string = self._format_args(action, default)
-                for option_string in action.option_strings:
-                    parts.append('%s %s' % (option_string, args_string))
-
-            return ', '.join(parts)
-
-    def _metavar_formatter(self, action, default_metavar):
-        if action.metavar is not None:
-            result = action.metavar
-        elif action.choices is not None:
-            choice_strs = [str(choice) for choice in action.choices]
-            result = '{%s}' % ','.join(choice_strs)
-        else:
-            result = default_metavar
-
-        def format(tuple_size):
-            if isinstance(result, tuple):
-                return result
-            else:
-                return (result, ) * tuple_size
-        return format
-
-    def _format_args(self, action, default_metavar):
-        get_metavar = self._metavar_formatter(action, default_metavar)
-        if action.nargs is None:
-            result = '%s' % get_metavar(1)
-        elif action.nargs == OPTIONAL:
-            result = '[%s]' % get_metavar(1)
-        elif action.nargs == ZERO_OR_MORE:
-            result = '[%s [%s ...]]' % get_metavar(2)
-        elif action.nargs == ONE_OR_MORE:
-            result = '%s [%s ...]' % get_metavar(2)
-        elif action.nargs == REMAINDER:
-            result = '...'
-        elif action.nargs == PARSER:
-            result = '%s ...' % get_metavar(1)
-        else:
-            formats = ['%s' for _ in range(action.nargs)]
-            result = ' '.join(formats) % get_metavar(action.nargs)
-        return result
-
-    def _expand_help(self, action):
-        params = dict(vars(action), prog=self._prog)
-        for name in list(params):
-            if params[name] is SUPPRESS:
-                del params[name]
-        for name in list(params):
-            if hasattr(params[name], '__name__'):
-                params[name] = params[name].__name__
-        if params.get('choices') is not None:
-            choices_str = ', '.join([str(c) for c in params['choices']])
-            params['choices'] = choices_str
-        return self._get_help_string(action) % params
-
-    def _iter_indented_subactions(self, action):
-        try:
-            get_subactions = action._get_subactions
-        except AttributeError:
-            pass
-        else:
-            self._indent()
-            for subaction in get_subactions():
-                yield subaction
-            self._dedent()
-
-    def _split_lines(self, text, width):
-        text = self._whitespace_matcher.sub(' ', text).strip()
-        return _textwrap.wrap(text, width)
-
-    def _fill_text(self, text, width, indent):
-        text = self._whitespace_matcher.sub(' ', text).strip()
-        return _textwrap.fill(text, width, initial_indent=indent,
-                                           subsequent_indent=indent)
-
-    def _get_help_string(self, action):
-        return action.help
-
-
-class RawDescriptionHelpFormatter(HelpFormatter):
-    """Help message formatter which retains any formatting in descriptions.
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _fill_text(self, text, width, indent):
-        return ''.join([indent + line for line in text.splitlines(True)])
-
-
-class RawTextHelpFormatter(RawDescriptionHelpFormatter):
-    """Help message formatter which retains formatting of all help text.
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _split_lines(self, text, width):
-        return text.splitlines()
-
-
-class ArgumentDefaultsHelpFormatter(HelpFormatter):
-    """Help message formatter which adds default values to argument help.
-
-    Only the name of this class is considered a public API. All the methods
-    provided by the class are considered an implementation detail.
-    """
-
-    def _get_help_string(self, action):
-        help = action.help
-        if '%(default)' not in action.help:
-            if action.default is not SUPPRESS:
-                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
-                if action.option_strings or action.nargs in defaulting_nargs:
-                    help += ' (default: %(default)s)'
-        return help
-
-
-# =====================
-# Options and Arguments
-# =====================
-
-def _get_action_name(argument):
-    if argument is None:
-        return None
-    elif argument.option_strings:
-        return  '/'.join(argument.option_strings)
-    elif argument.metavar not in (None, SUPPRESS):
-        return argument.metavar
-    elif argument.dest not in (None, SUPPRESS):
-        return argument.dest
-    else:
-        return None
-
-
-class ArgumentError(Exception):
-    """An error from creating or using an argument (optional or positional).
-
-    The string value of this exception is the message, augmented with
-    information about the argument that caused it.
-    """
-
-    def __init__(self, argument, message):
-        self.argument_name = _get_action_name(argument)
-        self.message = message
-
-    def __str__(self):
-        if self.argument_name is None:
-            format = '%(message)s'
-        else:
-            format = 'argument %(argument_name)s: %(message)s'
-        return format % dict(message=self.message,
-                             argument_name=self.argument_name)
-
-
-class ArgumentTypeError(Exception):
-    """An error from trying to convert a command line string to a type."""
-    pass
-
-
-# ==============
-# Action classes
-# ==============
-
-class Action(_AttributeHolder):
-    """Information about how to convert command line strings to Python objects.
-
-    Action objects are used by an ArgumentParser to represent the information
-    needed to parse a single argument from one or more strings from the
-    command line. The keyword arguments to the Action constructor are also
-    all attributes of Action instances.
-
-    Keyword Arguments:
-
-        - option_strings -- A list of command-line option strings which
-            should be associated with this action.
-
-        - dest -- The name of the attribute to hold the created object(s)
-
-        - nargs -- The number of command-line arguments that should be
-            consumed. By default, one argument will be consumed and a single
-            value will be produced.  Other values include:
-                - N (an integer) consumes N arguments (and produces a list)
-                - '?' consumes zero or one arguments
-                - '*' consumes zero or more arguments (and produces a list)
-                - '+' consumes one or more arguments (and produces a list)
-            Note that the difference between the default and nargs=1 is that
-            with the default, a single value will be produced, while with
-            nargs=1, a list containing a single value will be produced.
-
-        - const -- The value to be produced if the option is specified and the
-            option uses an action that takes no values.
-
-        - default -- The value to be produced if the option is not specified.
-
-        - type -- A callable that accepts a single string argument, and
-            returns the converted value.  The standard Python types str, int,
-            float, and complex are useful examples of such callables.  If None,
-            str is used.
-
-        - choices -- A container of values that should be allowed. If not None,
-            after a command-line argument has been converted to the appropriate
-            type, an exception will be raised if it is not a member of this
-            collection.
-
-        - required -- True if the action must always be specified at the
-            command line. This is only meaningful for optional command-line
-            arguments.
-
-        - help -- The help string describing the argument.
-
-        - metavar -- The name to be used for the option's argument with the
-            help string. If None, the 'dest' value will be used as the name.
-    """
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 nargs=None,
-                 const=None,
-                 default=None,
-                 type=None,
-                 choices=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-        self.option_strings = option_strings
-        self.dest = dest
-        self.nargs = nargs
-        self.const = const
-        self.default = default
-        self.type = type
-        self.choices = choices
-        self.required = required
-        self.help = help
-        self.metavar = metavar
-
-    def _get_kwargs(self):
-        names = [
-            'option_strings',
-            'dest',
-            'nargs',
-            'const',
-            'default',
-            'type',
-            'choices',
-            'help',
-            'metavar',
-        ]
-        return [(name, getattr(self, name)) for name in names]
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        raise NotImplementedError(_('.__call__() not defined'))
-
-
-class _StoreAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 nargs=None,
-                 const=None,
-                 default=None,
-                 type=None,
-                 choices=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-        if nargs == 0:
-            raise ValueError('nargs for store actions must be > 0; if you '
-                             'have nothing to store, actions such as store '
-                             'true or store const may be more appropriate')
-        if const is not None and nargs != OPTIONAL:
-            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
-        super(_StoreAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=nargs,
-            const=const,
-            default=default,
-            type=type,
-            choices=choices,
-            required=required,
-            help=help,
-            metavar=metavar)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        setattr(namespace, self.dest, values)
-
-
-class _StoreConstAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 const,
-                 default=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-        super(_StoreConstAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=0,
-            const=const,
-            default=default,
-            required=required,
-            help=help)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        setattr(namespace, self.dest, self.const)
-
-
-class _StoreTrueAction(_StoreConstAction):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 default=False,
-                 required=False,
-                 help=None):
-        super(_StoreTrueAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            const=True,
-            default=default,
-            required=required,
-            help=help)
-
-
-class _StoreFalseAction(_StoreConstAction):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 default=True,
-                 required=False,
-                 help=None):
-        super(_StoreFalseAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            const=False,
-            default=default,
-            required=required,
-            help=help)
-
-
-class _AppendAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 nargs=None,
-                 const=None,
-                 default=None,
-                 type=None,
-                 choices=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-        if nargs == 0:
-            raise ValueError('nargs for append actions must be > 0; if arg '
-                             'strings are not supplying the value to append, '
-                             'the append const action may be more appropriate')
-        if const is not None and nargs != OPTIONAL:
-            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
-        super(_AppendAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=nargs,
-            const=const,
-            default=default,
-            type=type,
-            choices=choices,
-            required=required,
-            help=help,
-            metavar=metavar)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        items = _copy.copy(_ensure_value(namespace, self.dest, []))
-        items.append(values)
-        setattr(namespace, self.dest, items)
-
-
-class _AppendConstAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 const,
-                 default=None,
-                 required=False,
-                 help=None,
-                 metavar=None):
-        super(_AppendConstAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=0,
-            const=const,
-            default=default,
-            required=required,
-            help=help,
-            metavar=metavar)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        items = _copy.copy(_ensure_value(namespace, self.dest, []))
-        items.append(self.const)
-        setattr(namespace, self.dest, items)
-
-
-class _CountAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest,
-                 default=None,
-                 required=False,
-                 help=None):
-        super(_CountAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=0,
-            default=default,
-            required=required,
-            help=help)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        new_count = _ensure_value(namespace, self.dest, 0) + 1
-        setattr(namespace, self.dest, new_count)
-
-
-class _HelpAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 dest=SUPPRESS,
-                 default=SUPPRESS,
-                 help=None):
-        super(_HelpAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            default=default,
-            nargs=0,
-            help=help)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        parser.print_help()
-        parser.exit()
-
-
-class _VersionAction(Action):
-
-    def __init__(self,
-                 option_strings,
-                 version=None,
-                 dest=SUPPRESS,
-                 default=SUPPRESS,
-                 help="show program's version number and exit"):
-        super(_VersionAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            default=default,
-            nargs=0,
-            help=help)
-        self.version = version
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        version = self.version
-        if version is None:
-            version = parser.version
-        formatter = parser._get_formatter()
-        formatter.add_text(version)
-        parser.exit(message=formatter.format_help())
-
-
-class _SubParsersAction(Action):
-
-    class _ChoicesPseudoAction(Action):
-
-        def __init__(self, name, aliases, help):
-            metavar = dest = name
-            if aliases:
-                metavar += ' (%s)' % ', '.join(aliases)
-            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
-            sup.__init__(option_strings=[], dest=dest, help=help,
-                         metavar=metavar)
-
-    def __init__(self,
-                 option_strings,
-                 prog,
-                 parser_class,
-                 dest=SUPPRESS,
-                 help=None,
-                 metavar=None):
-
-        self._prog_prefix = prog
-        self._parser_class = parser_class
-        self._name_parser_map = _collections.OrderedDict()
-        self._choices_actions = []
-
-        super(_SubParsersAction, self).__init__(
-            option_strings=option_strings,
-            dest=dest,
-            nargs=PARSER,
-            choices=self._name_parser_map,
-            help=help,
-            metavar=metavar)
-
-    def add_parser(self, name, **kwargs):
-        # set prog from the existing prefix
-        if kwargs.get('prog') is None:
-            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
-
-        aliases = kwargs.pop('aliases', ())
-
-        # create a pseudo-action to hold the choice help
-        if 'help' in kwargs:
-            help = kwargs.pop('help')
-            choice_action = self._ChoicesPseudoAction(name, aliases, help)
-            self._choices_actions.append(choice_action)
-
-        # create the parser and add it to the map
-        parser = self._parser_class(**kwargs)
-        self._name_parser_map[name] = parser
-
-        # make parser available under aliases also
-        for alias in aliases:
-            self._name_parser_map[alias] = parser
-
-        return parser
-
-    def _get_subactions(self):
-        return self._choices_actions
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        parser_name = values[0]
-        arg_strings = values[1:]
-
-        # set the parser name if requested
-        if self.dest is not SUPPRESS:
-            setattr(namespace, self.dest, parser_name)
-
-        # select the parser
-        try:
-            parser = self._name_parser_map[parser_name]
-        except KeyError:
-            args = {'parser_name': parser_name,
-                    'choices': ', '.join(self._name_parser_map)}
-            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
-            raise ArgumentError(self, msg)
-
-        # parse all the remaining options into the namespace
-        # store any unrecognized options on the object, so that the top
-        # level parser can decide what to do with them
-        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
-        if arg_strings:
-            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
-            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
-
-
-# ==============
-# Type classes
-# ==============
-
-class FileType(object):
-    """Factory for creating file object types
-
-    Instances of FileType are typically passed as type= arguments to the
-    ArgumentParser add_argument() method.
-
-    Keyword Arguments:
-        - mode -- A string indicating how the file is to be opened. Accepts the
-            same values as the builtin open() function.
-        - bufsize -- The file's desired buffer size. Accepts the same values as
-            the builtin open() function.
-    """
-
-    def __init__(self, mode='r', bufsize=-1):
-        self._mode = mode
-        self._bufsize = bufsize
-
-    def __call__(self, string):
-        # the special argument "-" means sys.std{in,out}
-        if string == '-':
-            if 'r' in self._mode:
-                return _sys.stdin
-            elif 'w' in self._mode:
-                return _sys.stdout
-            else:
-                msg = _('argument "-" with mode %r') % self._mode
-                raise ValueError(msg)
-
-        # all other arguments are used as file names
-        try:
-            return open(string, self._mode, self._bufsize)
-        except IOError as e:
-            message = _("can't open '%s': %s")
-            raise ArgumentTypeError(message % (string, e))
-
-    def __repr__(self):
-        args = self._mode, self._bufsize
-        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
-        return '%s(%s)' % (type(self).__name__, args_str)
-
-# ===========================
-# Optional and Positional Parsing
-# ===========================
-
-class Namespace(_AttributeHolder):
-    """Simple object for storing attributes.
-
-    Implements equality by attribute names and values, and provides a simple
-    string representation.
-    """
-
-    def __init__(self, **kwargs):
-        for name in kwargs:
-            setattr(self, name, kwargs[name])
-
-    def __eq__(self, other):
-        return vars(self) == vars(other)
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __contains__(self, key):
-        return key in self.__dict__
-
-
-class _ActionsContainer(object):
-
-    def __init__(self,
-                 description,
-                 prefix_chars,
-                 argument_default,
-                 conflict_handler):
-        super(_ActionsContainer, self).__init__()
-
-        self.description = description
-        self.argument_default = argument_default
-        self.prefix_chars = prefix_chars
-        self.conflict_handler = conflict_handler
-
-        # set up registries
-        self._registries = {}
-
-        # register actions
-        self.register('action', None, _StoreAction)
-        self.register('action', 'store', _StoreAction)
-        self.register('action', 'store_const', _StoreConstAction)
-        self.register('action', 'store_true', _StoreTrueAction)
-        self.register('action', 'store_false', _StoreFalseAction)
-        self.register('action', 'append', _AppendAction)
-        self.register('action', 'append_const', _AppendConstAction)
-        self.register('action', 'count', _CountAction)
-        self.register('action', 'help', _HelpAction)
-        self.register('action', 'version', _VersionAction)
-        self.register('action', 'parsers', _SubParsersAction)
-
-        # raise an exception if the conflict handler is invalid
-        self._get_handler()
-
-        # action storage
-        self._actions = []
-        self._option_string_actions = {}
-
-        # groups
-        self._action_groups = []
-        self._mutually_exclusive_groups = []
-
-        # defaults storage
-        self._defaults = {}
-
-        # determines whether an "option" looks like a negative number
-        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
-
-        # whether or not there are any optionals that look like negative
-        # numbers -- uses a list so it can be shared and edited
-        self._has_negative_number_optionals = []
-
-    # ====================
-    # Registration methods
-    # ====================
-    def register(self, registry_name, value, object):
-        registry = self._registries.setdefault(registry_name, {})
-        registry[value] = object
-
-    def _registry_get(self, registry_name, value, default=None):
-        return self._registries[registry_name].get(value, default)
-
-    # ==================================
-    # Namespace default accessor methods
-    # ==================================
-    def set_defaults(self, **kwargs):
-        self._defaults.update(kwargs)
-
-        # if these defaults match any existing arguments, replace
-        # the previous default on the object with the new one
-        for action in self._actions:
-            if action.dest in kwargs:
-                action.default = kwargs[action.dest]
-
-    def get_default(self, dest):
-        for action in self._actions:
-            if action.dest == dest and action.default is not None:
-                return action.default
-        return self._defaults.get(dest, None)
-
-
-    # =======================
-    # Adding argument actions
-    # =======================
-    def add_argument(self, *args, **kwargs):
-        """
-        add_argument(dest, ..., name=value, ...)
-        add_argument(option_string, option_string, ..., name=value, ...)
-        """
-
-        # if no positional args are supplied or only one is supplied and
-        # it doesn't look like an option string, parse a positional
-        # argument
-        chars = self.prefix_chars
-        if not args or len(args) == 1 and args[0][0] not in chars:
-            if args and 'dest' in kwargs:
-                raise ValueError('dest supplied twice for positional argument')
-            kwargs = self._get_positional_kwargs(*args, **kwargs)
-
-        # otherwise, we're adding an optional argument
-        else:
-            kwargs = self._get_optional_kwargs(*args, **kwargs)
-
-        # if no default was supplied, use the parser-level default
-        if 'default' not in kwargs:
-            dest = kwargs['dest']
-            if dest in self._defaults:
-                kwargs['default'] = self._defaults[dest]
-            elif self.argument_default is not None:
-                kwargs['default'] = self.argument_default
-
-        # create the action object, and add it to the parser
-        action_class = self._pop_action_class(kwargs)
-        if not callable(action_class):
-            raise ValueError('unknown action "%s"' % (action_class,))
-        action = action_class(**kwargs)
-
-        # raise an error if the action type is not callable
-        type_func = self._registry_get('type', action.type, action.type)
-        if not callable(type_func):
-            raise ValueError('%r is not callable' % (type_func,))
-
-        # raise an error if the metavar does not match the type
-        if hasattr(self, "_get_formatter"):
-            try:
-                self._get_formatter()._format_args(action, None)
-            except TypeError:
-                raise ValueError("length of metavar tuple does not match nargs")
-
-        return self._add_action(action)
-
-    def add_argument_group(self, *args, **kwargs):
-        group = _ArgumentGroup(self, *args, **kwargs)
-        self._action_groups.append(group)
-        return group
-
-    def add_mutually_exclusive_group(self, **kwargs):
-        group = _MutuallyExclusiveGroup(self, **kwargs)
-        self._mutually_exclusive_groups.append(group)
-        return group
-
-    def _add_action(self, action):
-        # resolve any conflicts
-        self._check_conflict(action)
-
-        # add to actions list
-        self._actions.append(action)
-        action.container = self
-
-        # index the action by any option strings it has
-        for option_string in action.option_strings:
-            self._option_string_actions[option_string] = action
-
-        # set the flag if any option strings look like negative numbers
-        for option_string in action.option_strings:
-            if self._negative_number_matcher.match(option_string):
-                if not self._has_negative_number_optionals:
-                    self._has_negative_number_optionals.append(True)
-
-        # return the created action
-        return action
-
-    def _remove_action(self, action):
-        self._actions.remove(action)
-
-    def _add_container_actions(self, container):
-        # collect groups by titles
-        title_group_map = {}
-        for group in self._action_groups:
-            if group.title in title_group_map:
-                msg = _('cannot merge actions - two groups are named %r')
-                raise ValueError(msg % (group.title))
-            title_group_map[group.title] = group
-
-        # map each action to its group
-        group_map = {}
-        for group in container._action_groups:
-
-            # if a group with the title exists, use that, otherwise
-            # create a new group matching the container's group
-            if group.title not in title_group_map:
-                title_group_map[group.title] = self.add_argument_group(
-                    title=group.title,
-                    description=group.description,
-                    conflict_handler=group.conflict_handler)
-
-            # map the actions to their new group
-            for action in group._group_actions:
-                group_map[action] = title_group_map[group.title]
-
-        # add container's mutually exclusive groups
-        # NOTE: if add_mutually_exclusive_group ever gains title= and
-        # description= then this code will need to be expanded as above
-        for group in container._mutually_exclusive_groups:
-            mutex_group = self.add_mutually_exclusive_group(
-                required=group.required)
-
-            # map the actions to their new mutex group
-            for action in group._group_actions:
-                group_map[action] = mutex_group
-
-        # add all actions to this container or their group
-        for action in container._actions:
-            group_map.get(action, self)._add_action(action)
-
-    def _get_positional_kwargs(self, dest, **kwargs):
-        # make sure required is not specified
-        if 'required' in kwargs:
-            msg = _("'required' is an invalid argument for positionals")
-            raise TypeError(msg)
-
-        # mark positional arguments as required if at least one is
-        # always required
-        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
-            kwargs['required'] = True
-        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
-            kwargs['required'] = True
-
-        # return the keyword arguments with no option strings
-        return dict(kwargs, dest=dest, option_strings=[])
-
-    def _get_optional_kwargs(self, *args, **kwargs):
-        # determine short and long option strings
-        option_strings = []
-        long_option_strings = []
-        for option_string in args:
-            # error on strings that don't start with an appropriate prefix
-            if not option_string[0] in self.prefix_chars:
-                args = {'option': option_string,
-                        'prefix_chars': self.prefix_chars}
-                msg = _('invalid option string %(option)r: '
-                        'must start with a character %(prefix_chars)r')
-                raise ValueError(msg % args)
-
-            # strings starting with two prefix characters are long options
-            option_strings.append(option_string)
-            if option_string[0] in self.prefix_chars:
-                if len(option_string) > 1:
-                    if option_string[1] in self.prefix_chars:
-                        long_option_strings.append(option_string)
-
-        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
-        dest = kwargs.pop('dest', None)
-        if dest is None:
-            if long_option_strings:
-                dest_option_string = long_option_strings[0]
-            else:
-                dest_option_string = option_strings[0]
-            dest = dest_option_string.lstrip(self.prefix_chars)
-            if not dest:
-                msg = _('dest= is required for options like %r')
-                raise ValueError(msg % option_string)
-            dest = dest.replace('-', '_')
-
-        # return the updated keyword arguments
-        return dict(kwargs, dest=dest, option_strings=option_strings)
-
-    def _pop_action_class(self, kwargs, default=None):
-        action = kwargs.pop('action', default)
-        return self._registry_get('action', action, action)
-
-    def _get_handler(self):
-        # determine function from conflict handler string
-        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
-        try:
-            return getattr(self, handler_func_name)
-        except AttributeError:
-            msg = _('invalid conflict_resolution value: %r')
-            raise ValueError(msg % self.conflict_handler)
-
-    def _check_conflict(self, action):
-
-        # find all options that conflict with this option
-        confl_optionals = []
-        for option_string in action.option_strings:
-            if option_string in self._option_string_actions:
-                confl_optional = self._option_string_actions[option_string]
-                confl_optionals.append((option_string, confl_optional))
-
-        # resolve any conflicts
-        if confl_optionals:
-            conflict_handler = self._get_handler()
-            conflict_handler(action, confl_optionals)
-
-    def _handle_conflict_error(self, action, conflicting_actions):
-        message = ngettext('conflicting option string: %s',
-                           'conflicting option strings: %s',
-                           len(conflicting_actions))
-        conflict_string = ', '.join([option_string
-                                     for option_string, action
-                                     in conflicting_actions])
-        raise ArgumentError(action, message % conflict_string)
-
-    def _handle_conflict_resolve(self, action, conflicting_actions):
-
-        # remove all conflicting options
-        for option_string, action in conflicting_actions:
-
-            # remove the conflicting option
-            action.option_strings.remove(option_string)
-            self._option_string_actions.pop(option_string, None)
-
-            # if the option now has no option string, remove it from the
-            # container holding it
-            if not action.option_strings:
-                action.container._remove_action(action)
-
-
-class _ArgumentGroup(_ActionsContainer):
-
-    def __init__(self, container, title=None, description=None, **kwargs):
-        # add any missing keyword arguments by checking the container
-        update = kwargs.setdefault
-        update('conflict_handler', container.conflict_handler)
-        update('prefix_chars', container.prefix_chars)
-        update('argument_default', container.argument_default)
-        super_init = super(_ArgumentGroup, self).__init__
-        super_init(description=description, **kwargs)
-
-        # group attributes
-        self.title = title
-        self._group_actions = []
-
-        # share most attributes with the container
-        self._registries = container._registries
-        self._actions = container._actions
-        self._option_string_actions = container._option_string_actions
-        self._defaults = container._defaults
-        self._has_negative_number_optionals = \
-            container._has_negative_number_optionals
-        self._mutually_exclusive_groups = container._mutually_exclusive_groups
-
-    def _add_action(self, action):
-        action = super(_ArgumentGroup, self)._add_action(action)
-        self._group_actions.append(action)
-        return action
-
-    def _remove_action(self, action):
-        super(_ArgumentGroup, self)._remove_action(action)
-        self._group_actions.remove(action)
-
-
-class _MutuallyExclusiveGroup(_ArgumentGroup):
-
-    def __init__(self, container, required=False):
-        super(_MutuallyExclusiveGroup, self).__init__(container)
-        self.required = required
-        self._container = container
-
-    def _add_action(self, action):
-        if action.required:
-            msg = _('mutually exclusive arguments must be optional')
-            raise ValueError(msg)
-        action = self._container._add_action(action)
-        self._group_actions.append(action)
-        return action
-
-    def _remove_action(self, action):
-        self._container._remove_action(action)
-        self._group_actions.remove(action)
-
-
-class ArgumentParser(_AttributeHolder, _ActionsContainer):
-    """Object for parsing command line strings into Python objects.
-
-    Keyword Arguments:
-        - prog -- The name of the program (default: sys.argv[0])
-        - usage -- A usage message (default: auto-generated from arguments)
-        - description -- A description of what the program does
-        - epilog -- Text following the argument descriptions
-        - parents -- Parsers whose arguments should be copied into this one
-        - formatter_class -- HelpFormatter class for printing help messages
-        - prefix_chars -- Characters that prefix optional arguments
-        - fromfile_prefix_chars -- Characters that prefix files containing
-            additional arguments
-        - argument_default -- The default value for all arguments
-        - conflict_handler -- String indicating how to handle conflicts
-        - add_help -- Add a -h/-help option
-    """
-
-    def __init__(self,
-                 prog=None,
-                 usage=None,
-                 description=None,
-                 epilog=None,
-                 version=None,
-                 parents=[],
-                 formatter_class=HelpFormatter,
-                 prefix_chars='-',
-                 fromfile_prefix_chars=None,
-                 argument_default=None,
-                 conflict_handler='error',
-                 add_help=True):
-
-        if version is not None:
-            import warnings
-            warnings.warn(
-                """The "version" argument to ArgumentParser is deprecated. """
-                """Please use """
-                """"add_argument(..., action='version', version="N", ...)" """
-                """instead""", DeprecationWarning)
-
-        superinit = super(ArgumentParser, self).__init__
-        superinit(description=description,
-                  prefix_chars=prefix_chars,
-                  argument_default=argument_default,
-                  conflict_handler=conflict_handler)
-
-        # default setting for prog
-        if prog is None:
-            prog = _os.path.basename(_sys.argv[0])
-
-        self.prog = prog
-        self.usage = usage
-        self.epilog = epilog
-        self.version = version
-        self.formatter_class = formatter_class
-        self.fromfile_prefix_chars = fromfile_prefix_chars
-        self.add_help = add_help
-
-        add_group = self.add_argument_group
-        self._positionals = add_group(_('positional arguments'))
-        self._optionals = add_group(_('optional arguments'))
-        self._subparsers = None
-
-        # register types
-        def identity(string):
-            return string
-        self.register('type', None, identity)
-
-        # add help and version arguments if necessary
-        # (using explicit default to override global argument_default)
-        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
-        if self.add_help:
-            self.add_argument(
-                default_prefix+'h', default_prefix*2+'help',
-                action='help', default=SUPPRESS,
-                help=_('show this help message and exit'))
-        if self.version:
-            self.add_argument(
-                default_prefix+'v', default_prefix*2+'version',
-                action='version', default=SUPPRESS,
-                version=self.version,
-                help=_("show program's version number and exit"))
-
-        # add parent arguments and defaults
-        for parent in parents:
-            self._add_container_actions(parent)
-            try:
-                defaults = parent._defaults
-            except AttributeError:
-                pass
-            else:
-                self._defaults.update(defaults)
-
-    # =======================
-    # Pretty __repr__ methods
-    # =======================
-    def _get_kwargs(self):
-        names = [
-            'prog',
-            'usage',
-            'description',
-            'version',
-            'formatter_class',
-            'conflict_handler',
-            'add_help',
-        ]
-        return [(name, getattr(self, name)) for name in names]
-
-    # ==================================
-    # Optional/Positional adding methods
-    # ==================================
-    def add_subparsers(self, **kwargs):
-        if self._subparsers is not None:
-            self.error(_('cannot have multiple subparser arguments'))
-
-        # add the parser class to the arguments if it's not present
-        kwargs.setdefault('parser_class', type(self))
-
-        if 'title' in kwargs or 'description' in kwargs:
-            title = _(kwargs.pop('title', 'subcommands'))
-            description = _(kwargs.pop('description', None))
-            self._subparsers = self.add_argument_group(title, description)
-        else:
-            self._subparsers = self._positionals
-
-        # prog defaults to the usage message of this parser, skipping
-        # optional arguments and with no "usage:" prefix
-        if kwargs.get('prog') is None:
-            formatter = self._get_formatter()
-            positionals = self._get_positional_actions()
-            groups = self._mutually_exclusive_groups
-            formatter.add_usage(self.usage, positionals, groups, '')
-            kwargs['prog'] = formatter.format_help().strip()
-
-        # create the parsers action and add it to the positionals list
-        parsers_class = self._pop_action_class(kwargs, 'parsers')
-        action = parsers_class(option_strings=[], **kwargs)
-        self._subparsers._add_action(action)
-
-        # return the created parsers action
-        return action
-
-    def _add_action(self, action):
-        if action.option_strings:
-            self._optionals._add_action(action)
-        else:
-            self._positionals._add_action(action)
-        return action
-
-    def _get_optional_actions(self):
-        return [action
-                for action in self._actions
-                if action.option_strings]
-
-    def _get_positional_actions(self):
-        return [action
-                for action in self._actions
-                if not action.option_strings]
-
-    # =====================================
-    # Command line argument parsing methods
-    # =====================================
-    def parse_args(self, args=None, namespace=None):
-        args, argv = self.parse_known_args(args, namespace)
-        if argv:
-            msg = _('unrecognized arguments: %s')
-            self.error(msg % ' '.join(argv))
-        return args
-
-    def parse_known_args(self, args=None, namespace=None):
-        if args is None:
-            # args default to the system args
-            args = _sys.argv[1:]
-        else:
-            # make sure that args are mutable
-            args = list(args)
-
-        # default Namespace built from parser defaults
-        if namespace is None:
-            namespace = Namespace()
-
-        # add any action defaults that aren't present
-        for action in self._actions:
-            if action.dest is not SUPPRESS:
-                if not hasattr(namespace, action.dest):
-                    if action.default is not SUPPRESS:
-                        setattr(namespace, action.dest, action.default)
-
-        # add any parser defaults that aren't present
-        for dest in self._defaults:
-            if not hasattr(namespace, dest):
-                setattr(namespace, dest, self._defaults[dest])
-
-        # parse the arguments and exit if there are any errors
-        try:
-            namespace, args = self._parse_known_args(args, namespace)
-            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
-                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
-                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
-            return namespace, args
-        except ArgumentError:
-            err = _sys.exc_info()[1]
-            self.error(str(err))
-
-    def _parse_known_args(self, arg_strings, namespace):
-        # replace arg strings that are file references
-        if self.fromfile_prefix_chars is not None:
-            arg_strings = self._read_args_from_files(arg_strings)
-
-        # map all mutually exclusive arguments to the other arguments
-        # they can't occur with
-        action_conflicts = {}
-        for mutex_group in self._mutually_exclusive_groups:
-            group_actions = mutex_group._group_actions
-            for i, mutex_action in enumerate(mutex_group._group_actions):
-                conflicts = action_conflicts.setdefault(mutex_action, [])
-                conflicts.extend(group_actions[:i])
-                conflicts.extend(group_actions[i + 1:])
-
-        # find all option indices, and determine the arg_string_pattern
-        # which has an 'O' if there is an option at an index,
-        # an 'A' if there is an argument, or a '-' if there is a '--'
-        option_string_indices = {}
-        arg_string_pattern_parts = []
-        arg_strings_iter = iter(arg_strings)
-        for i, arg_string in enumerate(arg_strings_iter):
-
-            # all args after -- are non-options
-            if arg_string == '--':
-                arg_string_pattern_parts.append('-')
-                for arg_string in arg_strings_iter:
-                    arg_string_pattern_parts.append('A')
-
-            # otherwise, add the arg to the arg strings
-            # and note the index if it was an option
-            else:
-                option_tuple = self._parse_optional(arg_string)
-                if option_tuple is None:
-                    pattern = 'A'
-                else:
-                    option_string_indices[i] = option_tuple
-                    pattern = 'O'
-                arg_string_pattern_parts.append(pattern)
-
-        # join the pieces together to form the pattern
-        arg_strings_pattern = ''.join(arg_string_pattern_parts)
-
-        # converts arg strings to the appropriate and then takes the action
-        seen_actions = set()
-        seen_non_default_actions = set()
-
-        def take_action(action, argument_strings, option_string=None):
-            seen_actions.add(action)
-            argument_values = self._get_values(action, argument_strings)
-
-            # error if this argument is not allowed with other previously
-            # seen arguments, assuming that actions that use the default
-            # value don't really count as "present"
-            if argument_values is not action.default:
-                seen_non_default_actions.add(action)
-                for conflict_action in action_conflicts.get(action, []):
-                    if conflict_action in seen_non_default_actions:
-                        msg = _('not allowed with argument %s')
-                        action_name = _get_action_name(conflict_action)
-                        raise ArgumentError(action, msg % action_name)
-
-            # take the action if we didn't receive a SUPPRESS value
-            # (e.g. from a default)
-            if argument_values is not SUPPRESS:
-                action(self, namespace, argument_values, option_string)
-
-        # function to convert arg_strings into an optional action
-        def consume_optional(start_index):
-
-            # get the optional identified at this index
-            option_tuple = option_string_indices[start_index]
-            action, option_string, explicit_arg = option_tuple
-
-            # identify additional optionals in the same arg string
-            # (e.g. -xyz is the same as -x -y -z if no args are required)
-            match_argument = self._match_argument
-            action_tuples = []
-            while True:
-
-                # if we found no optional action, skip it
-                if action is None:
-                    extras.append(arg_strings[start_index])
-                    return start_index + 1
-
-                # if there is an explicit argument, try to match the
-                # optional's string arguments to only this
-                if explicit_arg is not None:
-                    arg_count = match_argument(action, 'A')
-
-                    # if the action is a single-dash option and takes no
-                    # arguments, try to parse more single-dash options out
-                    # of the tail of the option string
-                    chars = self.prefix_chars
-                    if arg_count == 0 and option_string[1] not in chars:
-                        action_tuples.append((action, [], option_string))
-                        char = option_string[0]
-                        option_string = char + explicit_arg[0]
-                        new_explicit_arg = explicit_arg[1:] or None
-                        optionals_map = self._option_string_actions
-                        if option_string in optionals_map:
-                            action = optionals_map[option_string]
-                            explicit_arg = new_explicit_arg
-                        else:
-                            msg = _('ignored explicit argument %r')
-                            raise ArgumentError(action, msg % explicit_arg)
-
-                    # if the action expect exactly one argument, we've
-                    # successfully matched the option; exit the loop
-                    elif arg_count == 1:
-                        stop = start_index + 1
-                        args = [explicit_arg]
-                        action_tuples.append((action, args, option_string))
-                        break
-
-                    # error if a double-dash option did not use the
-                    # explicit argument
-                    else:
-                        msg = _('ignored explicit argument %r')
-                        raise ArgumentError(action, msg % explicit_arg)
-
-                # if there is no explicit argument, try to match the
-                # optional's string arguments with the following strings
-                # if successful, exit the loop
-                else:
-                    start = start_index + 1
-                    selected_patterns = arg_strings_pattern[start:]
-                    arg_count = match_argument(action, selected_patterns)
-                    stop = start + arg_count
-                    args = arg_strings[start:stop]
-                    action_tuples.append((action, args, option_string))
-                    break
-
-            # add the Optional to the list and return the index at which
-            # the Optional's string args stopped
-            assert action_tuples
-            for action, args, option_string in action_tuples:
-                take_action(action, args, option_string)
-            return stop
-
-        # the list of Positionals left to be parsed; this is modified
-        # by consume_positionals()
-        positionals = self._get_positional_actions()
-
-        # function to convert arg_strings into positional actions
-        def consume_positionals(start_index):
-            # match as many Positionals as possible
-            match_partial = self._match_arguments_partial
-            selected_pattern = arg_strings_pattern[start_index:]
-            arg_counts = match_partial(positionals, selected_pattern)
-
-            # slice off the appropriate arg strings for each Positional
-            # and add the Positional and its args to the list
-            for action, arg_count in zip(positionals, arg_counts):
-                args = arg_strings[start_index: start_index + arg_count]
-                start_index += arg_count
-                take_action(action, args)
-
-            # slice off the Positionals that we just parsed and return the
-            # index at which the Positionals' string args stopped
-            positionals[:] = positionals[len(arg_counts):]
-            return start_index
-
-        # consume Positionals and Optionals alternately, until we have
-        # passed the last option string
-        extras = []
-        start_index = 0
-        if option_string_indices:
-            max_option_string_index = max(option_string_indices)
-        else:
-            max_option_string_index = -1
-        while start_index <= max_option_string_index:
-
-            # consume any Positionals preceding the next option
-            next_option_string_index = min([
-                index
-                for index in option_string_indices
-                if index >= start_index])
-            if start_index != next_option_string_index:
-                positionals_end_index = consume_positionals(start_index)
-
-                # only try to parse the next optional if we didn't consume
-                # the option string during the positionals parsing
-                if positionals_end_index > start_index:
-                    start_index = positionals_end_index
-                    continue
-                else:
-                    start_index = positionals_end_index
-
-            # if we consumed all the positionals we could and we're not
-            # at the index of an option string, there were extra arguments
-            if start_index not in option_string_indices:
-                strings = arg_strings[start_index:next_option_string_index]
-                extras.extend(strings)
-                start_index = next_option_string_index
-
-            # consume the next optional and any arguments for it
-            start_index = consume_optional(start_index)
-
-        # consume any positionals following the last Optional
-        stop_index = consume_positionals(start_index)
-
-        # if we didn't consume all the argument strings, there were extras
-        extras.extend(arg_strings[stop_index:])
-
-        # if we didn't use all the Positional objects, there were too few
-        # arg strings supplied.
-        if positionals:
-            self.error(_('too few arguments'))
-
-        # make sure all required actions were present, and convert defaults.
-        for action in self._actions:
-            if action not in seen_actions:
-                if action.required:
-                    name = _get_action_name(action)
-                    self.error(_('argument %s is required') % name)
-                else:
-                    # Convert action default now instead of doing it before
-                    # parsing arguments to avoid calling convert functions
-                    # twice (which may fail) if the argument was given, but
-                    # only if it was defined already in the namespace
-                    if (action.default is not None and
-                            isinstance(action.default, str) and
-                            hasattr(namespace, action.dest) and
-                            action.default is getattr(namespace, action.dest)):
-                        setattr(namespace, action.dest,
-                                self._get_value(action, action.default))
-
-        # make sure all required groups had one option present
-        for group in self._mutually_exclusive_groups:
-            if group.required:
-                for action in group._group_actions:
-                    if action in seen_non_default_actions:
-                        break
-
-                # if no actions were used, report the error
-                else:
-                    names = [_get_action_name(action)
-                             for action in group._group_actions
-                             if action.help is not SUPPRESS]
-                    msg = _('one of the arguments %s is required')
-                    self.error(msg % ' '.join(names))
-
-        # return the updated namespace and the extra arguments
-        return namespace, extras
-
-    def _read_args_from_files(self, arg_strings):
-        # expand arguments referencing files
-        new_arg_strings = []
-        for arg_string in arg_strings:
-
-            # for regular arguments, just add them back into the list
-            if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
-                new_arg_strings.append(arg_string)
-
-            # replace arguments referencing files with the file content
-            else:
-                try:
-                    args_file = open(arg_string[1:])
-                    try:
-                        arg_strings = []
-                        for arg_line in args_file.read().splitlines():
-                            for arg in self.convert_arg_line_to_args(arg_line):
-                                arg_strings.append(arg)
-                        arg_strings = self._read_args_from_files(arg_strings)
-                        new_arg_strings.extend(arg_strings)
-                    finally:
-                        args_file.close()
-                except IOError:
-                    err = _sys.exc_info()[1]
-                    self.error(str(err))
-
-        # return the modified argument list
-        return new_arg_strings
-
-    def convert_arg_line_to_args(self, arg_line):
-        return [arg_line]
-
-    def _match_argument(self, action, arg_strings_pattern):
-        # match the pattern for this action to the arg strings
-        nargs_pattern = self._get_nargs_pattern(action)
-        match = _re.match(nargs_pattern, arg_strings_pattern)
-
-        # raise an exception if we weren't able to find a match
-        if match is None:
-            nargs_errors = {
-                None: _('expected one argument'),
-                OPTIONAL: _('expected at most one argument'),
-                ONE_OR_MORE: _('expected at least one argument'),
-            }
-            default = ngettext('expected %s argument',
-                               'expected %s arguments',
-                               action.nargs) % action.nargs
-            msg = nargs_errors.get(action.nargs, default)
-            raise ArgumentError(action, msg)
-
-        # return the number of arguments matched
-        return len(match.group(1))
-
-    def _match_arguments_partial(self, actions, arg_strings_pattern):
-        # progressively shorten the actions list by slicing off the
-        # final actions until we find a match
-        result = []
-        for i in range(len(actions), 0, -1):
-            actions_slice = actions[:i]
-            pattern = ''.join([self._get_nargs_pattern(action)
-                               for action in actions_slice])
-            match = _re.match(pattern, arg_strings_pattern)
-            if match is not None:
-                result.extend([len(string) for string in match.groups()])
-                break
-
-        # return the list of arg string counts
-        return result
-
-    def _parse_optional(self, arg_string):
-        # if it's an empty string, it was meant to be a positional
-        if not arg_string:
-            return None
-
-        # if it doesn't start with a prefix, it was meant to be positional
-        if not arg_string[0] in self.prefix_chars:
-            return None
-
-        # if the option string is present in the parser, return the action
-        if arg_string in self._option_string_actions:
-            action = self._option_string_actions[arg_string]
-            return action, arg_string, None
-
-        # if it's just a single character, it was meant to be positional
-        if len(arg_string) == 1:
-            return None
-
-        # if the option string before the "=" is present, return the action
-        if '=' in arg_string:
-            option_string, explicit_arg = arg_string.split('=', 1)
-            if option_string in self._option_string_actions:
-                action = self._option_string_actions[option_string]
-                return action, option_string, explicit_arg
-
-        # search through all possible prefixes of the option string
-        # and all actions in the parser for possible interpretations
-        option_tuples = self._get_option_tuples(arg_string)
-
-        # if multiple actions match, the option string was ambiguous
-        if len(option_tuples) > 1:
-            options = ', '.join([option_string
-                for action, option_string, explicit_arg in option_tuples])
-            args = {'option': arg_string, 'matches': options}
-            msg = _('ambiguous option: %(option)s could match %(matches)s')
-            self.error(msg % args)
-
-        # if exactly one action matched, this segmentation is good,
-        # so return the parsed action
-        elif len(option_tuples) == 1:
-            option_tuple, = option_tuples
-            return option_tuple
-
-        # if it was not found as an option, but it looks like a negative
-        # number, it was meant to be positional
-        # unless there are negative-number-like options
-        if self._negative_number_matcher.match(arg_string):
-            if not self._has_negative_number_optionals:
-                return None
-
-        # if it contains a space, it was meant to be a positional
-        if ' ' in arg_string:
-            return None
-
-        # it was meant to be an optional but there is no such option
-        # in this parser (though it might be a valid option in a subparser)
-        return None, arg_string, None
-
-    def _get_option_tuples(self, option_string):
-        result = []
-
-        # option strings starting with two prefix characters are only
-        # split at the '='
-        chars = self.prefix_chars
-        if option_string[0] in chars and option_string[1] in chars:
-            if '=' in option_string:
-                option_prefix, explicit_arg = option_string.split('=', 1)
-            else:
-                option_prefix = option_string
-                explicit_arg = None
-            for option_string in self._option_string_actions:
-                if option_string.startswith(option_prefix):
-                    action = self._option_string_actions[option_string]
-                    tup = action, option_string, explicit_arg
-                    result.append(tup)
-
-        # single character options can be concatenated with their arguments
-        # but multiple character options always have to have their argument
-        # separate
-        elif option_string[0] in chars and option_string[1] not in chars:
-            option_prefix = option_string
-            explicit_arg = None
-            short_option_prefix = option_string[:2]
-            short_explicit_arg = option_string[2:]
-
-            for option_string in self._option_string_actions:
-                if option_string == short_option_prefix:
-                    action = self._option_string_actions[option_string]
-                    tup = action, option_string, short_explicit_arg
-                    result.append(tup)
-                elif option_string.startswith(option_prefix):
-                    action = self._option_string_actions[option_string]
-                    tup = action, option_string, explicit_arg
-                    result.append(tup)
-
-        # shouldn't ever get here
-        else:
-            self.error(_('unexpected option string: %s') % option_string)
-
-        # return the collected option tuples
-        return result
-
-    def _get_nargs_pattern(self, action):
-        # in all examples below, we have to allow for '--' args
-        # which are represented as '-' in the pattern
-        nargs = action.nargs
-
-        # the default (None) is assumed to be a single argument
-        if nargs is None:
-            nargs_pattern = '(-*A-*)'
-
-        # allow zero or one arguments
-        elif nargs == OPTIONAL:
-            nargs_pattern = '(-*A?-*)'
-
-        # allow zero or more arguments
-        elif nargs == ZERO_OR_MORE:
-            nargs_pattern = '(-*[A-]*)'
-
-        # allow one or more arguments
-        elif nargs == ONE_OR_MORE:
-            nargs_pattern = '(-*A[A-]*)'
-
-        # allow any number of options or arguments
-        elif nargs == REMAINDER:
-            nargs_pattern = '([-AO]*)'
-
-        # allow one argument followed by any number of options or arguments
-        elif nargs == PARSER:
-            nargs_pattern = '(-*A[-AO]*)'
-
-        # all others should be integers
-        else:
-            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
-
-        # if this is an optional action, -- is not allowed
-        if action.option_strings:
-            nargs_pattern = nargs_pattern.replace('-*', '')
-            nargs_pattern = nargs_pattern.replace('-', '')
-
-        # return the pattern
-        return nargs_pattern
-
-    # ========================
-    # Value conversion methods
-    # ========================
-    def _get_values(self, action, arg_strings):
-        # for everything but PARSER, REMAINDER args, strip out first '--'
-        if action.nargs not in [PARSER, REMAINDER]:
-            try:
-                arg_strings.remove('--')
-            except ValueError:
-                pass
-
-        # optional argument produces a default when not present
-        if not arg_strings and action.nargs == OPTIONAL:
-            if action.option_strings:
-                value = action.const
-            else:
-                value = action.default
-            if isinstance(value, str):
-                value = self._get_value(action, value)
-                self._check_value(action, value)
-
-        # when nargs='*' on a positional, if there were no command-line
-        # args, use the default if it is anything other than None
-        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
-              not action.option_strings):
-            if action.default is not None:
-                value = action.default
-            else:
-                value = arg_strings
-            self._check_value(action, value)
-
-        # single argument or optional argument produces a single value
-        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
-            arg_string, = arg_strings
-            value = self._get_value(action, arg_string)
-            self._check_value(action, value)
-
-        # REMAINDER arguments convert all values, checking none
-        elif action.nargs == REMAINDER:
-            value = [self._get_value(action, v) for v in arg_strings]
-
-        # PARSER arguments convert all values, but check only the first
-        elif action.nargs == PARSER:
-            value = [self._get_value(action, v) for v in arg_strings]
-            self._check_value(action, value[0])
-
-        # all other types of nargs produce a list
-        else:
-            value = [self._get_value(action, v) for v in arg_strings]
-            for v in value:
-                self._check_value(action, v)
-
-        # return the converted value
-        return value
-
-    def _get_value(self, action, arg_string):
-        type_func = self._registry_get('type', action.type, action.type)
-        if not callable(type_func):
-            msg = _('%r is not callable')
-            raise ArgumentError(action, msg % type_func)
-
-        # convert the value to the appropriate type
-        try:
-            result = type_func(arg_string)
-
-        # ArgumentTypeErrors indicate errors
-        except ArgumentTypeError:
-            name = getattr(action.type, '__name__', repr(action.type))
-            msg = str(_sys.exc_info()[1])
-            raise ArgumentError(action, msg)
-
-        # TypeErrors or ValueErrors also indicate errors
-        except (TypeError, ValueError):
-            name = getattr(action.type, '__name__', repr(action.type))
-            args = {'type': name, 'value': arg_string}
-            msg = _('invalid %(type)s value: %(value)r')
-            raise ArgumentError(action, msg % args)
-
-        # return the converted value
-        return result
-
-    def _check_value(self, action, value):
-        # converted value must be one of the choices (if specified)
-        if action.choices is not None and value not in action.choices:
-            args = {'value': value,
-                    'choices': ', '.join(map(repr, action.choices))}
-            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
-            raise ArgumentError(action, msg % args)
-
-    # =======================
-    # Help-formatting methods
-    # =======================
-    def format_usage(self):
-        formatter = self._get_formatter()
-        formatter.add_usage(self.usage, self._actions,
-                            self._mutually_exclusive_groups)
-        return formatter.format_help()
-
-    def format_help(self):
-        formatter = self._get_formatter()
-
-        # usage
-        formatter.add_usage(self.usage, self._actions,
-                            self._mutually_exclusive_groups)
-
-        # description
-        formatter.add_text(self.description)
-
-        # positionals, optionals and user-defined groups
-        for action_group in self._action_groups:
-            formatter.start_section(action_group.title)
-            formatter.add_text(action_group.description)
-            formatter.add_arguments(action_group._group_actions)
-            formatter.end_section()
-
-        # epilog
-        formatter.add_text(self.epilog)
-
-        # determine help from format above
-        return formatter.format_help()
-
-    def format_version(self):
-        import warnings
-        warnings.warn(
-            'The format_version method is deprecated -- the "version" '
-            'argument to ArgumentParser is no longer supported.',
-            DeprecationWarning)
-        formatter = self._get_formatter()
-        formatter.add_text(self.version)
-        return formatter.format_help()
-
-    def _get_formatter(self):
-        return self.formatter_class(prog=self.prog)
-
-    # =====================
-    # Help-printing methods
-    # =====================
-    def print_usage(self, file=None):
-        if file is None:
-            file = _sys.stdout
-        self._print_message(self.format_usage(), file)
-
-    def print_help(self, file=None):
-        if file is None:
-            file = _sys.stdout
-        self._print_message(self.format_help(), file)
-
-    def print_version(self, file=None):
-        import warnings
-        warnings.warn(
-            'The print_version method is deprecated -- the "version" '
-            'argument to ArgumentParser is no longer supported.',
-            DeprecationWarning)
-        self._print_message(self.format_version(), file)
-
-    def _print_message(self, message, file=None):
-        if message:
-            if file is None:
-                file = _sys.stderr
-            file.write(message)
-
-    # ===============
-    # Exiting methods
-    # ===============
-    def exit(self, status=0, message=None):
-        if message:
-            self._print_message(message, _sys.stderr)
-        _sys.exit(status)
-
-    def error(self, message):
-        """error(message: string)
-
-        Prints a usage message incorporating the message to stderr and
-        exits.
-
-        If you override this in a subclass, it should not return -- it
-        should either exit or raise an exception.
-        """
-        self.print_usage(_sys.stderr)
-        args = {'prog': self.prog, 'message': message}
-        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)

borg/testsuite/__init__.py

@@ -7,7 +7,6 @@ import sys
 import sysconfig
 import time
 import unittest
-from ..helpers import st_mtime_ns
 from ..xattr import get_all
 
 try:
@@ -31,9 +30,6 @@ else:
 if sys.platform.startswith('netbsd'):
     st_mtime_ns_round = -4  # only >1 microsecond resolution here?
 
-has_mtime_ns = sys.version >= '3.3'
-utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
-
 
 class BaseTestCase(unittest.TestCase):
     """
@@ -80,14 +76,13 @@ class BaseTestCase(unittest.TestCase):
                 d1[4] = None
             if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]):
                 d2[4] = None
-            if not os.path.islink(path1) or utime_supports_fd:
-                # Older versions of llfuse do not support ns precision properly
-                if fuse and not have_fuse_mtime_ns:
-                    d1.append(round(st_mtime_ns(s1), -4))
-                    d2.append(round(st_mtime_ns(s2), -4))
-                else:
-                    d1.append(round(st_mtime_ns(s1), st_mtime_ns_round))
-                    d2.append(round(st_mtime_ns(s2), st_mtime_ns_round))
+            # Older versions of llfuse do not support ns precision properly
+            if fuse and not have_fuse_mtime_ns:
+                d1.append(round(s1.st_mtime_ns, -4))
+                d2.append(round(s2.st_mtime_ns, -4))
+            else:
+                d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
+                d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
             d1.append(get_all(path1, follow_symlinks=False))
             d2.append(get_all(path2, follow_symlinks=False))
             self.assert_equal(d1, d2)
@@ -149,4 +144,4 @@ class FakeInputs:
         try:
             return self.inputs.pop(0)
         except IndexError:
-            raise EOFError
+            raise EOFError from None

borg/testsuite/archive.py

@@ -1,7 +1,7 @@
 from datetime import datetime, timezone
+from unittest.mock import Mock
 
 import msgpack
-from mock import Mock
 
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker
 from ..key import PlaintextKey

borg/testsuite/archiver.py

@@ -11,9 +11,9 @@ import shutil
 import tempfile
 import time
 import unittest
+from unittest.mock import patch
 from hashlib import sha256
 
-from mock import patch
 import pytest
 
 from .. import xattr
@@ -21,7 +21,7 @@ from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP
 from ..archiver import Archiver
 from ..cache import Cache
 from ..crypto import bytes_to_long, num_aes_blocks
-from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, st_atime_ns, st_mtime_ns
+from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import BaseTestCase, changedir, environment_variable
@@ -34,13 +34,7 @@ except ImportError:
 
 has_lchflags = hasattr(os, 'lchflags')
 
-src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..')
-
-# Python <= 3.2 raises OSError instead of PermissionError (See #164)
-try:
-    PermissionError = PermissionError
-except NameError:
-    PermissionError = OSError
+src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
 
 def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
@@ -76,9 +70,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
 try:
     exec_cmd('help', exe='borg.exe', fork=True)
     BORG_EXES = ['python', 'binary', ]
-except (IOError, OSError) as err:
-    if err.errno != errno.ENOENT:
-        raise
+except FileNotFoundError:
     BORG_EXES = ['python', ]
 
 
@@ -100,7 +92,7 @@ def test_return_codes(cmd, tmpdir):
     input = tmpdir.mkdir('input')
     output = tmpdir.mkdir('output')
     input.join('test_file').write('content')
-    rc, out = cmd('init', '%s' % str(repo))
+    rc, out = cmd('init', '--encryption=none', '%s' % str(repo))
     assert rc == EXIT_SUCCESS
     rc, out = cmd('create', '%s::archive' % repo, str(input))
     assert rc == EXIT_SUCCESS
@@ -144,7 +136,7 @@ def test_disk_full(cmd):
                 data = os.urandom(size)
                 f.write(data)
 
-    with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='1'):
+    with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='YES'):
         mount = DF_MOUNT
         assert os.path.exists(mount)
         repo = os.path.join(mount, 'repo')
@@ -198,8 +190,9 @@ class ArchiverTestCaseBase(BaseTestCase):
     prefix = ''
 
     def setUp(self):
-        os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
-        os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1'
+        os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES'
+        os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES'
+        os.environ['BORG_PASSPHRASE'] = 'waytooeasyonlyfortests'
         self.archiver = not self.FORK_DEFAULT and Archiver() or None
         self.tmpdir = tempfile.mkdtemp()
         self.repository_path = os.path.join(self.tmpdir, 'repository')
@@ -337,7 +330,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         item_count = 3 if has_lchflags else 4  # one file is UF_NODUMP
         self.assert_in('Number of files: %d' % item_count, info_output)
         shutil.rmtree(self.cache_path)
-        with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='1'):
+        with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'):
             info_output2 = self.cmd('info', self.repository_location + '::test')
 
         def filter(output):
@@ -364,12 +357,12 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             self.cmd('extract', self.repository_location + '::test')
         sti = os.stat('input/file1')
         sto = os.stat('output/input/file1')
-        assert st_mtime_ns(sti) == st_mtime_ns(sto) == mtime * 1e9
+        assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9
         if hasattr(os, 'O_NOATIME'):
-            assert st_atime_ns(sti) == st_atime_ns(sto) == atime * 1e9
+            assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9
         else:
             # it touched the input file's atime while backing it up
-            assert st_atime_ns(sto) == atime * 1e9
+            assert sto.st_atime_ns == atime * 1e9
 
     def _extract_repository_id(self, path):
         return Repository(self.repository_path).id
@@ -433,7 +426,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
     def test_repository_swap_detection(self):
         self.create_test_files()
         os.environ['BORG_PASSPHRASE'] = 'passphrase'
-        self.cmd('init', '--encryption=passphrase', self.repository_location)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
         repository_id = self._extract_repository_id(self.repository_path)
         self.cmd('create', self.repository_location + '::test', 'input')
         shutil.rmtree(self.repository_path)
@@ -449,7 +442,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.create_test_files()
         self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted')
         os.environ['BORG_PASSPHRASE'] = 'passphrase'
-        self.cmd('init', '--encryption=passphrase', self.repository_location + '_encrypted')
+        self.cmd('init', '--encryption=repokey', self.repository_location + '_encrypted')
         self.cmd('create', self.repository_location + '_encrypted::test', 'input')
         shutil.rmtree(self.repository_path + '_encrypted')
         os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted')
@@ -829,14 +822,12 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         output = self.cmd('create', '-v', '--list', '--filter=AM', self.repository_location + '::test3', 'input')
         self.assert_in('file1', output)
 
-    def test_cmdline_compatibility(self):
-        self.create_regular_file('file1', size=1024 * 80)
-        self.cmd('init', self.repository_location)
-        self.cmd('create', self.repository_location + '::test', 'input')
-        output = self.cmd('verify', '-v', self.repository_location + '::test')
-        self.assert_in('"borg verify" has been deprecated', output)
-        output = self.cmd('prune', self.repository_location, '--hourly=1')
-        self.assert_in('"--hourly" has been deprecated. Use "--keep-hourly" instead', output)
+    #def test_cmdline_compatibility(self):
+    #    self.create_regular_file('file1', size=1024 * 80)
+    #    self.cmd('init', self.repository_location)
+    #    self.cmd('create', self.repository_location + '::test', 'input')
+    #    output = self.cmd('foo', self.repository_location, '--old')
+    #    self.assert_in('"--old" has been deprecated. Use "--new" instead', output)
 
     def test_prune_repository(self):
         self.cmd('init', self.repository_location)
@@ -993,7 +984,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.verify_aes_counter_uniqueness('keyfile')
 
     def test_aes_counter_uniqueness_passphrase(self):
-        self.verify_aes_counter_uniqueness('passphrase')
+        self.verify_aes_counter_uniqueness('repokey')
 
     def test_debug_dump_archive_items(self):
         self.create_test_files()

borg/testsuite/benchmark.py

@@ -16,16 +16,16 @@ from .archiver import changedir, cmd
 @pytest.yield_fixture
 def repo_url(request, tmpdir):
     os.environ['BORG_PASSPHRASE'] = '123456'
-    os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
-    os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1'
-    os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1'
+    os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES'
+    os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES'
+    os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = 'yes'
     os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys'))
     os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache'))
     yield str(tmpdir.join('repository'))
     tmpdir.remove(rec=1)
 
 
-@pytest.fixture(params=["none", "passphrase"])
+@pytest.fixture(params=["none", "repokey"])
 def repo(request, cmd, repo_url):
     cmd('init', '--encryption', request.param, repo_url)
     return repo_url
@@ -40,13 +40,10 @@ def testdata(request, tmpdir_factory):
         # do not use a binary zero (\0) to avoid sparse detection
         data = lambda: b'0' * size
     if data_type == 'random':
-        rnd = open('/dev/urandom', 'rb')
-        data = lambda: rnd.read(size)
+        data = lambda: os.urandom(size)
     for i in range(count):
         with open(str(p.join(str(i))), "wb") as f:
             f.write(data())
-    if data_type == 'random':
-        rnd.close()
     yield str(p)
     p.remove(rec=1)
 

borg/testsuite/crypto.py

@@ -1,6 +1,6 @@
 from binascii import hexlify
 
-from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, pbkdf2_sha256, get_random_bytes
+from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes
 from . import BaseTestCase
 
 
@@ -13,21 +13,6 @@ class CryptoTestCase(BaseTestCase):
         self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1)
         self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1')
 
-    def test_pbkdf2_sha256(self):
-        self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 1, 32)),
-                          b'120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b')
-        self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 2, 32)),
-                          b'ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43')
-        self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 4096, 32)),
-                          b'c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a')
-
-    def test_get_random_bytes(self):
-        bytes = get_random_bytes(10)
-        bytes2 = get_random_bytes(10)
-        self.assert_equal(len(bytes), 10)
-        self.assert_equal(len(bytes2), 10)
-        self.assert_not_equal(bytes, bytes2)
-
     def test_aes(self):
         key = b'X' * 32
         data = b'foo' * 10

borg/testsuite/helpers.py

@@ -9,11 +9,11 @@ import sys
 import msgpack
 import msgpack.fallback
 
-from ..helpers import Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \
-    prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \
+from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \
+    prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, TRUISH, FALSISH, DEFAULTISH, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
-    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher, \
-    ShellPattern
+    ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \
+    PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
 from . import BaseTestCase, environment_variable, FakeInputs
 
 
@@ -458,11 +458,6 @@ def test_pattern_matcher():
 def test_compression_specs():
     with pytest.raises(ValueError):
         CompressionSpec('')
-    assert CompressionSpec('0') == dict(name='zlib', level=0)
-    assert CompressionSpec('1') == dict(name='zlib', level=1)
-    assert CompressionSpec('9') == dict(name='zlib', level=9)
-    with pytest.raises(ValueError):
-        CompressionSpec('10')
     assert CompressionSpec('none') == dict(name='none')
     assert CompressionSpec('lz4') == dict(name='lz4')
     assert CompressionSpec('zlib') == dict(name='zlib', level=6)
@@ -691,20 +686,28 @@ def test_is_slow_msgpack():
     assert not is_slow_msgpack()
 
 
-def test_yes_simple():
-    input = FakeInputs(['y', 'Y', 'yes', 'Yes', ])
-    assert yes(input=input)
-    assert yes(input=input)
-    assert yes(input=input)
-    assert yes(input=input)
-    input = FakeInputs(['n', 'N', 'no', 'No', ])
-    assert not yes(input=input)
-    assert not yes(input=input)
-    assert not yes(input=input)
-    assert not yes(input=input)
+def test_yes_input():
+    inputs = list(TRUISH)
+    input = FakeInputs(inputs)
+    for i in inputs:
+        assert yes(input=input)
+    inputs = list(FALSISH)
+    input = FakeInputs(inputs)
+    for i in inputs:
+        assert not yes(input=input)
+
+
+def test_yes_input_defaults():
+    inputs = list(DEFAULTISH)
+    input = FakeInputs(inputs)
+    for i in inputs:
+        assert yes(default=True, input=input)
+    input = FakeInputs(inputs)
+    for i in inputs:
+        assert not yes(default=False, input=input)
 
 
-def test_yes_custom():
+def test_yes_input_custom():
     input = FakeInputs(['YES', 'SURE', 'NOPE', ])
     assert yes(truish=('YES', ), input=input)
     assert yes(truish=('SURE', ), input=input)
@@ -712,11 +715,20 @@ def test_yes_custom():
 
 
 def test_yes_env():
-    input = FakeInputs(['n', 'n'])
-    with environment_variable(OVERRIDE_THIS='nonempty'):
-        assert yes(env_var_override='OVERRIDE_THIS', input=input)
-    with environment_variable(OVERRIDE_THIS=None):  # env not set
-        assert not yes(env_var_override='OVERRIDE_THIS', input=input)
+    for value in TRUISH:
+        with environment_variable(OVERRIDE_THIS=value):
+            assert yes(env_var_override='OVERRIDE_THIS')
+    for value in FALSISH:
+        with environment_variable(OVERRIDE_THIS=value):
+            assert not yes(env_var_override='OVERRIDE_THIS')
+
+
+def test_yes_env_default():
+    for value in DEFAULTISH:
+        with environment_variable(OVERRIDE_THIS=value):
+            assert yes(env_var_override='OVERRIDE_THIS', default=True)
+        with environment_variable(OVERRIDE_THIS=value):
+            assert not yes(env_var_override='OVERRIDE_THIS', default=False)
 
 
 def test_yes_defaults():
@@ -728,27 +740,27 @@ def test_yes_defaults():
     assert yes(default=True, input=input)
     assert yes(default=True, input=input)
     assert yes(default=True, input=input)
-    ifile = StringIO()
-    assert yes(default_notty=True, ifile=ifile)
-    assert not yes(default_notty=False, ifile=ifile)
     input = FakeInputs([])
-    assert yes(default_eof=True, input=input)
-    assert not yes(default_eof=False, input=input)
+    assert yes(default=True, input=input)
+    assert not yes(default=False, input=input)
     with pytest.raises(ValueError):
         yes(default=None)
-    with pytest.raises(ValueError):
-        yes(default_notty='invalid')
-    with pytest.raises(ValueError):
-        yes(default_eof='invalid')
 
 
 def test_yes_retry():
-    input = FakeInputs(['foo', 'bar', 'y', ])
+    input = FakeInputs(['foo', 'bar', TRUISH[0], ])
     assert yes(retry_msg='Retry: ', input=input)
-    input = FakeInputs(['foo', 'bar', 'N', ])
+    input = FakeInputs(['foo', 'bar', FALSISH[0], ])
     assert not yes(retry_msg='Retry: ', input=input)
 
 
+def test_yes_no_retry():
+    input = FakeInputs(['foo', 'bar', TRUISH[0], ])
+    assert not yes(retry=False, default=False, input=input)
+    input = FakeInputs(['foo', 'bar', FALSISH[0], ])
+    assert yes(retry=False, default=True, input=input)
+
+
 def test_yes_output(capfd):
     input = FakeInputs(['invalid', 'y', 'n'])
     assert yes(msg='intro-msg', false_msg='false-msg', true_msg='true-msg', retry_msg='retry-msg', input=input)

borg/testsuite/key.py

@@ -2,11 +2,11 @@ import os
 import re
 import shutil
 import tempfile
-from binascii import hexlify
+from binascii import hexlify, unhexlify
 
 from ..crypto import bytes_to_long, num_aes_blocks
 from ..key import PlaintextKey, PassphraseKey, KeyfileKey
-from ..helpers import Location, unhexlify
+from ..helpers import Location
 from . import BaseTestCase
 
 

borg/testsuite/repository.py

@@ -2,8 +2,7 @@ import os
 import shutil
 import sys
 import tempfile
-
-from mock import patch
+from unittest.mock import patch
 
 from ..hashindex import NSIndex
 from ..helpers import Location, IntegrityError

borg/xattr.py

@@ -231,9 +231,6 @@ elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
         mv = memoryview(namebuf.raw)
         while mv:
             length = mv[0]
-            # Python < 3.3 returns bytes instead of int
-            if isinstance(length, bytes):
-                length = ord(length)
             names.append(os.fsdecode(bytes(mv[1:1+length])))
             mv = mv[1+length:]
         return names

docs/development.rst

@@ -60,7 +60,7 @@ Some more advanced examples::
   # verify a changed tox.ini (run this after any change to tox.ini):
   fakeroot -u tox --recreate
 
-  fakeroot -u tox -e py32  # run all tests, but only on python 3.2
+  fakeroot -u tox -e py34  # run all tests, but only on python 3.4
 
   fakeroot -u tox borg.testsuite.locking  # only run 1 test module
 

docs/faq.rst

@@ -37,7 +37,7 @@ Which file types, attributes, etc. are preserved?
     * FIFOs ("named pipes")
     * Name
     * Contents
-    * Time of last modification (nanosecond precision with Python >= 3.3)
+    * Time of last modification (nanosecond precision)
     * IDs of owning user and owning group
     * Names of owning user and owning group (if the IDs can be resolved)
     * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)

docs/installation.rst

@@ -98,7 +98,7 @@ Dependencies
 To install |project_name| from a source package (including pip), you have to install the
 following dependencies first:
 
-* `Python 3`_ >= 3.2.2. Even though Python 3 is not the default Python version on
+* `Python 3`_ >= 3.4.0. Even though Python 3 is not the default Python version on
   most systems, it is usually available as an optional install.
 * OpenSSL_ >= 1.0.0
 * libacl_ (that pulls in libattr_ also)

docs/internals.rst

@@ -210,9 +210,9 @@ producing chunks of 2^HASH_MASK_BITS Bytes on average.
 ``borg create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE``
 can be used to tune the chunker parameters, the default is:
 
-- CHUNK_MIN_EXP = 10 (minimum chunk size = 2^10 B = 1 kiB)
+- CHUNK_MIN_EXP = 19 (minimum chunk size = 2^19 B = 512 kiB)
 - CHUNK_MAX_EXP = 23 (maximum chunk size = 2^23 B = 8 MiB)
-- HASH_MASK_BITS = 16 (statistical medium chunk size ~= 2^16 B = 64 kiB)
+- HASH_MASK_BITS = 21 (statistical medium chunk size ~= 2^21 B = 2 MiB)
 - HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`)
 
 The buzhash table is altered by XORing it with a seed randomly generated once
@@ -313,13 +313,13 @@ If a remote repository is used the repo index will be allocated on the remote si
 
 E.g. backing up a total count of 1 Mi (IEC binary prefix e.g. 2^20) files with a total size of 1TiB.
 
-a) with create ``--chunker-params 10,23,16,4095`` (default):
+a) with ``create --chunker-params 10,23,16,4095`` (custom, like borg < 1.0 or attic):
 
   mem_usage  =  2.8GiB
 
-b) with create ``--chunker-params 10,23,20,4095`` (custom):
+b) with ``create --chunker-params 19,23,21,4095`` (default):
 
-  mem_usage  =  0.4GiB
+  mem_usage  =  0.31GiB
 
 .. note:: There is also the ``--no-files-cache`` option to switch off the files cache.
    You'll save some memory, but it will need to read / chunk all the files as
@@ -344,7 +344,12 @@ To reduce payload size, only 8 bytes of the 16 bytes nonce is saved in the
 payload, the first 8 bytes are always zeros. This does not affect security but
 limits the maximum repository capacity to only 295 exabytes (2**64 * 16 bytes).
 
-Encryption keys are either derived from a passphrase or kept in a key file.
+Encryption keys (and other secrets) are kept either in a key file on the client
+('keyfile' mode) or in the repository config on the server ('repokey' mode).
+In both cases, the secrets are generated from random and then encrypted by a
+key derived from your passphrase (this happens on the client before the key
+is stored into the keyfile or as repokey).
+
 The passphrase is passed through the ``BORG_PASSPHRASE`` environment variable
 or prompted for interactive usage.
 

docs/misc/create_chunker-params.txt

@@ -6,7 +6,7 @@ About borg create --chunker-params
 CHUNK_MIN_EXP and CHUNK_MAX_EXP give the exponent N of the 2^N minimum and
 maximum chunk size. Required: CHUNK_MIN_EXP < CHUNK_MAX_EXP.
 
-Defaults: 10 (2^10 == 1KiB) minimum, 23 (2^23 == 8MiB) maximum.
+Defaults: 19 (2^19 == 512KiB) minimum, 23 (2^23 == 8MiB) maximum.
 
 HASH_MASK_BITS is the number of least-significant bits of the rolling hash
 that need to be zero to trigger a chunk cut.
@@ -14,7 +14,7 @@ Recommended: CHUNK_MIN_EXP + X <= HASH_MASK_BITS <= CHUNK_MAX_EXP - X, X >= 2
 (this allows the rolling hash some freedom to make its cut at a place
 determined by the windows contents rather than the min/max. chunk size).
 
-Default: 16 (statistically, chunks will be about 2^16 == 64kiB in size)
+Default: 21 (statistically, chunks will be about 2^21 == 2MiB in size)
 
 HASH_WINDOW_SIZE: the size of the window used for the rolling hash computation.
 Default: 4095B

docs/quickstart.rst

@@ -146,9 +146,10 @@ Keep an eye on CPU load and throughput.
 Repository encryption
 ---------------------
 
-Repository encryption is enabled at repository creation time::
+Repository encryption can be enabled or disabled at repository creation time
+(the default is enabled, with `repokey` method)::
 
-    $ borg init --encryption=repokey|keyfile PATH
+    $ borg init --encryption=none|repokey|keyfile PATH
 
 When repository encryption is enabled all data is encrypted using 256-bit AES_
 encryption and the integrity and authenticity is verified using `HMAC-SHA256`_.

docs/usage.rst

@@ -69,15 +69,19 @@ General:
     TMPDIR
         where temporary files are stored (might need a lot of temporary space for some operations)
 
-Some "yes" sayers (if set, they automatically confirm that you really want to do X even if there is that warning):
-    BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK
+Some automatic "answerers" (if set, they automatically answer confirmation questions):
+    BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes)
         For "Warning: Attempting to access a previously unknown unencrypted repository"
-    BORG_RELOCATED_REPO_ACCESS_IS_OK
+    BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes)
         For "Warning: The repository at location ... was previously located at ..."
-    BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
-        For "Warning: '``check --repair``' is an experimental feature that might result in data loss."
-    BORG_DELETE_I_KNOW_WHAT_I_AM_DOING
-        For "You requested to completely DELETE the repository *including* all archives it contains:  "
+    BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+        For "Warning: 'check --repair' is an experimental feature that might result in data loss."
+    BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES)
+        For "You requested to completely DELETE the repository *including* all archives it contains:"
+
+    Note: answers are case sensitive. setting an invalid answer value might either give the default
+    answer or ask you interactively, depending on whether retries are allowed (they by default are
+    allowed). So please test your scripts interactively before making them a non-interactive script.
 
 Directories:
     BORG_KEYS_DIR
@@ -100,7 +104,7 @@ Please note:
   (e.g. mode 600, root:root).
 
 
-.. _INI: https://docs.python.org/3.2/library/logging.config.html#configuration-file-format
+.. _INI: https://docs.python.org/3.4/library/logging.config.html#configuration-file-format
 
 Resource Usage
 ~~~~~~~~~~~~~~
@@ -194,12 +198,7 @@ an attacker has access to your backup repository.
 
 But be careful with the key / the passphrase:
 
-``--encryption=passphrase`` is DEPRECATED and will be removed in next major release.
-This mode has very fundamental, unfixable problems (like you can never change
-your passphrase or the pbkdf2 iteration count for an existing repository, because
-the encryption / decryption key is directly derived from the passphrase).
-
-If you want "passphrase-only" security, just use the ``repokey`` mode. The key will
+If you want "passphrase-only" security, use the ``repokey`` mode. The key will
 be stored inside the repository (in its "config" file). In above mentioned
 attack scenario, the attacker will have the key (but not the passphrase).
 
@@ -216,8 +215,10 @@ The backup that is encrypted with that key won't help you with that, of course.
 Make sure you use a good passphrase. Not too short, not too simple. The real
 encryption / decryption key is encrypted with / locked by your passphrase.
 If an attacker gets your key, he can't unlock and use it without knowing the
-passphrase. In ``repokey`` and ``keyfile`` modes, you can change your passphrase
-for existing repos.
+passphrase.
+
+You can change your passphrase for existing repos at any time, it won't affect
+the encryption/decryption key or other secrets.
 
 
 .. include:: usage/create.rst.inc
@@ -249,8 +250,10 @@ Examples
     NAME="root-`date +%Y-%m-%d`"
     $ borg create -C zlib,6 /mnt/backup::$NAME / --do-not-cross-mountpoints
 
-    # Backup huge files with little chunk management overhead
-    $ borg create --chunker-params 19,23,21,4095 /mnt/backup::VMs /srv/VMs
+    # Make a big effort in fine granular deduplication (big chunk management
+    # overhead, needs a lot of RAM and disk space, see formula in internals
+    # docs - same parameters as borg < 1.0 or attic):
+    $ borg create --chunker-params 10,23,16,4095 /mnt/backup::small /smallstuff
 
     # Backup a raw device (must not be active/in use/mounted at that time)
     $ dd if=/dev/sda bs=10M | borg create /mnt/backup::my-sda -
@@ -506,15 +509,15 @@ resource usage (RAM and disk space) as the amount of resources needed is
 (also) determined by the total amount of chunks in the repository (see
 `Indexes / Caches memory usage` for details).
 
-``--chunker-params=10,23,16,4095 (default)`` results in a fine-grained deduplication
-and creates a big amount of chunks and thus uses a lot of resources to manage them.
-This is good for relatively small data volumes and if the machine has a good
-amount of free RAM and disk space.
+``--chunker-params=10,23,16,4095`` results in a fine-grained deduplication
+and creates a big amount of chunks and thus uses a lot of resources to manage
+them. This is good for relatively small data volumes and if the machine has a
+good amount of free RAM and disk space.
 
-``--chunker-params=19,23,21,4095`` results in a coarse-grained deduplication and
-creates a much smaller amount of chunks and thus uses less resources.
-This is good for relatively big data volumes and if the machine has a relatively
-low amount of free RAM and disk space.
+``--chunker-params=19,23,21,4095`` (default) results in a coarse-grained
+deduplication and creates a much smaller amount of chunks and thus uses less
+resources. This is good for relatively big data volumes and if the machine has
+a relatively low amount of free RAM and disk space.
 
 If you already have made some archives in a repository and you then change
 chunker params, this of course impacts deduplication as the chunks will be

requirements.d/development.txt

@@ -1,7 +1,6 @@
 virtualenv<14.0
 tox
-mock
 pytest
-pytest-cov<2.0.0
-pytest-benchmark>=3.0.0
+pytest-cov
+pytest-benchmark
 Cython

setup.py

@@ -7,7 +7,7 @@ from glob import glob
 from distutils.command.build import build
 from distutils.core import Command
 
-min_python = (3, 2)
+min_python = (3, 4)
 my_python = sys.version_info
 
 if my_python < min_python:
@@ -211,7 +211,7 @@ if not on_rtd:
     Extension('borg.chunker', [chunker_source]),
     Extension('borg.hashindex', [hashindex_source])
 ]
-    if sys.platform.startswith('linux'):
+    if sys.platform == 'linux':
         ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
     elif sys.platform.startswith('freebsd'):
         ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
@@ -242,14 +242,12 @@ setup(
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.2',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
     ],
-    packages=['borg', 'borg.testsuite', 'borg.support', ],
+    packages=['borg', 'borg.testsuite', ],
     entry_points={
         'console_scripts': [
             'borg = borg.archiver:main',

tox.ini

@@ -2,7 +2,7 @@
 # fakeroot -u tox --recreate
 
 [tox]
-envlist = py{32,33,34,35}
+envlist = py{34,35}
 
 [testenv]
 # Change dir to avoid import problem for cython code. The directory does