Merge branch '1.0-maint' into merge/1.0-maint

# Conflicts: ... everywhere ...
#	.travis.yml
#	Vagrantfile
#	borg/testsuite/key.py
#	docs/changes.rst
#	docs/quickstart.rst
#	docs/usage.rst
#	docs/usage/upgrade.rst.inc
#	src/borg/archive.py
#	src/borg/archiver.py
#	src/borg/crypto.pyx
#	src/borg/helpers.py
#	src/borg/key.py
#	src/borg/remote.py
#	src/borg/repository.py
#	src/borg/testsuite/archive.py
#	src/borg/testsuite/archiver.py
#	src/borg/testsuite/crypto.py
#	src/borg/testsuite/helpers.py
#	src/borg/testsuite/repository.py
#	src/borg/upgrader.py
#	tox.ini

.travis.yml

@@ -17,7 +17,7 @@ matrix:
           os: linux
           dist: trusty
           env: TOXENV=py35
-        - python: 3.6-dev
+        - python: 3.6
           os: linux
           dist: trusty
           env: TOXENV=py36
@@ -33,6 +33,10 @@ matrix:
           os: osx
           osx_image: xcode6.4
           env: TOXENV=py35
+        - language: generic
+          os: osx
+          osx_image: xcode6.4
+          env: TOXENV=py36
     allow_failures:
         - os: osx
 

.travis/install.sh

@@ -18,7 +18,7 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
     brew install xz  # required for python lzma module
     brew outdated pyenv || brew upgrade pyenv
     brew install pkg-config
-    brew install Caskroom/versions/osxfuse
+    brew install Caskroom/cask/osxfuse
 
     case "${TOXENV}" in
         py34)
@@ -29,6 +29,10 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
             pyenv install 3.5.1
             pyenv global 3.5.1
             ;;
+        py36)
+            pyenv install 3.6.0
+            pyenv global 3.6.0
+            ;;
     esac
     pyenv rehash
     python -m pip install --user virtualenv

Vagrantfile

@@ -224,6 +224,7 @@ def install_pythons(boxname)
     . ~/.bash_profile
     pyenv install 3.4.0  # tests
     pyenv install 3.5.0  # tests
+    pyenv install 3.6.0  # tests
     pyenv install 3.5.2  # binary build, use latest 3.5.x release
     pyenv rehash
   EOF
@@ -317,8 +318,8 @@ def run_tests(boxname)
     . ../borg-env/bin/activate
     if which pyenv 2> /dev/null; then
       # for testing, use the earliest point releases of the supported python versions:
-      pyenv global 3.4.0 3.5.0
-      pyenv local 3.4.0 3.5.0
+      pyenv global 3.4.0 3.5.0 3.6.0
+      pyenv local 3.4.0 3.5.0 3.6.0
     fi
     # otherwise: just use the system python
     if which fakeroot 2> /dev/null; then

docs/changes.rst

@@ -204,6 +204,43 @@ Other changes:
   - point XDG_*_HOME to temp dirs for tests, #1714
   - remove all BORG_* env vars from the outer environment
 
+
+Version 1.0.10rc1 (not released yet)
+------------------------------------
+
+Bug fixes:
+
+- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992
+- When running out of buffer memory when reading xattrs, only skip the
+  current file, #1993
+- Fixed "borg upgrade --tam" crashing with unencrypted repositories. Since
+  :ref:`the issue <tam_vuln>` is not relevant for unencrypted repositories,
+  it now does nothing and prints an error, #1981.
+- Fixed change-passphrase crashing with unencrypted repositories, #1978
+- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997
+- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled
+
+Other changes:
+
+- xattr: ignore empty names returned by llistxattr(2) et al
+- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT,
+  SIGBUS and SIGILL signals to dump the Python traceback.
+- Also print a traceback on SIGUSR2.
+- borg change-passphrase: print key location (simplify making a backup of it)
+- officially support Python 3.6 (setup.py: add Python 3.6 qualifier)
+- tests:
+
+  - vagrant / travis / tox: add Python 3.6 based testing
+  - travis: fix osxfuse install (fixes OS X testing on Travis CI)
+  - use pytest-xdist to parallelize testing
+- docs:
+
+  - language clarification - VM backup FAQ
+- fix typos (taken from Debian package patch)
+- remote: include data hexdump in "unexpected RPC data" error message
+- remote: log SSH command line at debug level
+
+
 Version 1.0.9 (2016-12-20)
 --------------------------
 

docs/faq.rst

@@ -13,7 +13,7 @@ Yes, the `deduplication`_ technique used by
 Also, we have optional simple sparse file support for extract.
 
 If you use non-snapshotting backup tools like Borg to back up virtual machines,
-then these should be turned off for doing so. Backing up live VMs this way can (and will)
+then the VMs should be turned off for the duration of the backup. Backing up live VMs can (and will)
 result in corrupted or inconsistent backup contents: a VM image is just a regular file to
 Borg with the same issues as regular files when it comes to concurrent reading and writing from
 the same file.

docs/usage.rst

@@ -164,7 +164,7 @@ General:
     BORG_FILES_CACHE_TTL
         When set to a numeric value, this determines the maximum "time to live" for the files cache
         entries (default: 20). The files cache is used to quickly determine whether a file is unchanged.
-        The FAQ explains this more detailled in: :ref:`always_chunking`
+        The FAQ explains this more detailed in: :ref:`always_chunking`
     TMPDIR
         where temporary files are stored (might need a lot of temporary space for some operations)
 

requirements.d/development.txt

@@ -1,6 +1,7 @@
 virtualenv
 tox
 pytest
+pytest-xdist
 pytest-cov
 pytest-benchmark
 Cython

setup.py

@@ -409,6 +409,7 @@ setup(
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
     ],

src/borg/archive.py

@@ -1360,16 +1360,18 @@ class ArchiveChecker:
             sort_by = sort_by.split(',')
             if any((first, last, prefix)):
                 archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last)
+                if not archive_infos:
+                    logger.warning('--first/--last/--prefix did not match any archives')
             else:
                 archive_infos = self.manifest.archives.list(sort_by=sort_by)
         else:
             # we only want one specific archive
-            info = self.manifest.archives.get(archive)
-            if info is None:
+            try:
+                archive_infos = [self.manifest.archives[archive]]
+            except KeyError:
                 logger.error("Archive '%s' not found.", archive)
-                archive_infos = []
-            else:
-                archive_infos = [info]
+                self.error_found = True
+                return
         num_archives = len(archive_infos)
 
         with cache_if_remote(self.repository) as repository:

src/borg/archiver.py

@@ -1,5 +1,6 @@
 import argparse
 import collections
+import faulthandler
 import functools
 import hashlib
 import inspect
@@ -240,8 +241,14 @@ class Archiver:
     @with_repository()
     def do_change_passphrase(self, args, repository, manifest, key):
         """Change repository key file passphrase"""
+        if not hasattr(key, 'change_passphrase'):
+            print('This repository is not encrypted, cannot change the passphrase.')
+            return EXIT_ERROR
         key.change_passphrase()
         logger.info('Key updated')
+        if hasattr(key, 'find_key'):
+            # print key location to make backing it up easier
+            logger.info('Key location: %s', key.find_key())
         return EXIT_SUCCESS
 
     @with_repository(lock=False, exclusive=False, manifest=False, cache=False)
@@ -1078,6 +1085,10 @@ class Archiver:
         if args.tam:
             manifest, key = Manifest.load(repository, force_tam_not_required=args.force)
 
+            if not hasattr(key, 'change_passphrase'):
+                print('This repository is not encrypted, cannot enable TAM.')
+                return EXIT_ERROR
+
             if not manifest.tam_verified or not manifest.config.get(b'tam_required', False):
                 # The standard archive listing doesn't include the archive ID like in borg 1.1.x
                 print('Manifest contents:')
@@ -2390,7 +2401,7 @@ class Archiver:
         Upgrade an existing Borg repository.
 
         Borg 1.x.y upgrades
-        -------------------
+        +++++++++++++++++++
 
         Use ``borg upgrade --tam REPO`` to require manifest authentication
         introduced with Borg 1.0.9 to address security issues. This means
@@ -2412,7 +2423,7 @@ class Archiver:
         for details.
 
         Attic and Borg 0.xx to Borg 1.x
-        -------------------------------
+        +++++++++++++++++++++++++++++++
 
         This currently supports converting an Attic repository to Borg and also
         helps with converting Borg 0.xx to 1.0.
@@ -2852,6 +2863,11 @@ def sig_info_handler(sig_no, stack):  # pragma: no cover
                 break
 
 
+def sig_trace_handler(sig_no, stack):  # pragma: no cover
+    print('\nReceived SIGUSR2 at %s, dumping trace...' % datetime.now().replace(microsecond=0), file=sys.stderr)
+    faulthandler.dump_traceback()
+
+
 def main():  # pragma: no cover
     # provide 'borg mount' behaviour when the main script/executable is named borgfs
     if os.path.basename(sys.argv[0]) == "borgfs":
@@ -2868,10 +2884,14 @@ def main():  # pragma: no cover
     # SIGHUP is important especially for systemd systems, where logind
     # sends it when a session exits, in addition to any traditional use.
     # Output some info if we receive SIGUSR1 or SIGINFO (ctrl-t).
+
+    # Register fault handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL.
+    faulthandler.enable()
     with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \
          signal_handler('SIGHUP', raising_signal_handler(SigHup)), \
          signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \
          signal_handler('SIGUSR1', sig_info_handler), \
+         signal_handler('SIGUSR2', sig_trace_handler), \
          signal_handler('SIGINFO', sig_info_handler):
         archiver = Archiver()
         msg = tb = None

src/borg/helpers.py

@@ -60,9 +60,15 @@ class Error(Exception):
     # show a traceback?
     traceback = False
 
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.args = args
+
     def get_message(self):
         return type(self).__doc__.format(*self.args)
 
+    __str__ = get_message
+
 
 class ErrorWithTraceback(Error):
     """like Error, but show a traceback also"""
@@ -798,6 +804,10 @@ class Buffer:
     """
     provide a thread-local buffer
     """
+
+    class MemoryLimitExceeded(Error, OSError):
+        """Requested buffer size {} is above the limit of {}."""
+
     def __init__(self, allocator, size=4096, limit=None):
         """
         Initialize the buffer: use allocator(size) call to allocate a buffer.
@@ -817,11 +827,11 @@ class Buffer:
         """
         resize the buffer - to avoid frequent reallocation, we usually always grow (if needed).
         giving init=True it is possible to first-time initialize or shrink the buffer.
-        if a buffer size beyond the limit is requested, raise ValueError.
+        if a buffer size beyond the limit is requested, raise Buffer.MemoryLimitExceeded (OSError).
         """
         size = int(size)
         if self.limit is not None and size > self.limit:
-            raise ValueError('Requested buffer size %d is above the limit of %d.' % (size, self.limit))
+            raise Buffer.MemoryLimitExceeded(size, self.limit)
         if init or len(self) < size:
             self._thread_local.buffer = self.allocator(size)
 

src/borg/remote.py

@@ -10,6 +10,7 @@ import sys
 import tempfile
 import time
 import traceback
+import textwrap
 from subprocess import Popen, PIPE
 
 import msgpack
@@ -23,6 +24,9 @@ from .helpers import replace_placeholders
 from .helpers import yes
 from .repository import Repository
 from .version import parse_version, format_version
+from .logger import create_logger
+
+logger = create_logger(__name__)
 
 RPC_PROTOCOL_VERSION = 2
 BORG_VERSION = parse_version(__version__)
@@ -56,7 +60,16 @@ class UnexpectedRPCDataFormatFromClient(Error):
 
 
 class UnexpectedRPCDataFormatFromServer(Error):
-    """Got unexpected RPC data format from server."""
+    """Got unexpected RPC data format from server:\n{}"""
+
+    def __init__(self, data):
+        try:
+            data = data.decode()[:128]
+        except UnicodeDecodeError:
+            data = data[:128]
+            data = ['%02X' % byte for byte in data]
+            data = textwrap.fill(' '.join(data), 16 * 3)
+        super().__init__(data)
 
 
 # Protocol compatibility:
@@ -476,6 +489,7 @@ class RemoteRepository:
                 env.pop(lp_key, None)
         env.pop('BORG_PASSPHRASE', None)  # security: do not give secrets to subprocess
         env['BORG_VERSION'] = __version__
+        logger.debug('SSH command line: %s', borg_cmd)
         self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
         self.stdin_fd = self.p.stdin.fileno()
         self.stdout_fd = self.p.stdout.fileno()
@@ -685,7 +699,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
                             else:
                                 unpacked = {MSGID: msgid, RESULT: res}
                         else:
-                            raise UnexpectedRPCDataFormatFromServer()
+                            raise UnexpectedRPCDataFormatFromServer(data)
                         if msgid in self.ignore_responses:
                             self.ignore_responses.remove(msgid)
                             if b'exception_class' in unpacked:

src/borg/testsuite/archive.py

@@ -1,4 +1,5 @@
 import os
+from collections import OrderedDict
 from datetime import datetime, timezone
 from io import StringIO
 from unittest.mock import Mock
@@ -201,11 +202,15 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized):
     assert not valid_msgpacked_dict(packed, item_keys_serialized)
 
 
+# pytest-xdist requires always same order for the keys and dicts:
+IK = sorted(list(ITEM_KEYS))
+
+
 @pytest.mark.parametrize('packed',
     [msgpack.packb(o) for o in [
         {b'path': b'/a/b/c'},  # small (different msgpack mapping type!)
-        dict((k, b'') for k in ITEM_KEYS),  # as big (key count) as it gets
-        dict((k, b'x' * 1000) for k in ITEM_KEYS),  # as big (key count and volume) as it gets
+        OrderedDict((k, b'') for k in IK),  # as big (key count) as it gets
+        OrderedDict((k, b'x' * 1000) for k in IK),  # as big (key count and volume) as it gets
     ]])
 def test_valid_msgpacked_items(packed, item_keys_serialized):
     assert valid_msgpacked_dict(packed, item_keys_serialized)

src/borg/testsuite/archiver.py

@@ -2358,6 +2358,82 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
         assert not self.cmd('list', self.repository_location)
 
 
+class ManifestAuthenticationTest(ArchiverTestCaseBase):
+    def spoof_manifest(self, repository):
+        with repository:
+            _, key = Manifest.load(repository)
+            repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({
+                'version': 1,
+                'archives': {},
+                'config': {},
+                'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
+            })))
+            repository.commit()
+
+    def test_fresh_init_tam_required(self):
+        self.cmd('init', self.repository_location)
+        repository = Repository(self.repository_path, exclusive=True)
+        with repository:
+            manifest, key = Manifest.load(repository)
+            repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({
+                'version': 1,
+                'archives': {},
+                'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
+            })))
+            repository.commit()
+
+        with pytest.raises(TAMRequiredError):
+            self.cmd('list', self.repository_location)
+
+    def test_not_required(self):
+        self.cmd('init', self.repository_location)
+        self.create_src_archive('archive1234')
+        repository = Repository(self.repository_path, exclusive=True)
+        with repository:
+            shutil.rmtree(get_security_dir(bin_to_hex(repository.id)))
+            _, key = Manifest.load(repository)
+            key.tam_required = False
+            key.change_passphrase(key._passphrase)
+
+            manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID)))
+            del manifest[b'tam']
+            repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest)))
+            repository.commit()
+        output = self.cmd('list', '--debug', self.repository_location)
+        assert 'archive1234' in output
+        assert 'TAM not found and not required' in output
+        # Run upgrade
+        self.cmd('upgrade', '--tam', self.repository_location)
+        # Manifest must be authenticated now
+        output = self.cmd('list', '--debug', self.repository_location)
+        assert 'archive1234' in output
+        assert 'TAM-verified manifest' in output
+        # Try to spoof / modify pre-1.0.9
+        self.spoof_manifest(repository)
+        # Fails
+        with pytest.raises(TAMRequiredError):
+            self.cmd('list', self.repository_location)
+        # Force upgrade
+        self.cmd('upgrade', '--tam', '--force', self.repository_location)
+        self.cmd('list', self.repository_location)
+
+    def test_disable(self):
+        self.cmd('init', self.repository_location)
+        self.create_src_archive('archive1234')
+        self.cmd('upgrade', '--disable-tam', self.repository_location)
+        repository = Repository(self.repository_path, exclusive=True)
+        self.spoof_manifest(repository)
+        assert not self.cmd('list', self.repository_location)
+
+    def test_disable2(self):
+        self.cmd('init', self.repository_location)
+        self.create_src_archive('archive1234')
+        repository = Repository(self.repository_path, exclusive=True)
+        self.spoof_manifest(repository)
+        self.cmd('upgrade', '--disable-tam', self.repository_location)
+        assert not self.cmd('list', self.repository_location)
+
+
 @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteArchiverTestCase(ArchiverTestCase):
     prefix = '__testsuite__:'

src/borg/testsuite/helpers.py

@@ -783,7 +783,7 @@ class TestBuffer:
         buffer = Buffer(bytearray, size=100, limit=200)
         buffer.resize(200)
         assert len(buffer) == 200
-        with pytest.raises(ValueError):
+        with pytest.raises(Buffer.MemoryLimitExceeded):
             buffer.resize(201)
         assert len(buffer) == 200
 
@@ -797,7 +797,7 @@ class TestBuffer:
         b3 = buffer.get(200)
         assert len(b3) == 200
         assert b3 is not b2  # new, resized buffer
-        with pytest.raises(ValueError):
+        with pytest.raises(Buffer.MemoryLimitExceeded):
             buffer.get(201)  # beyond limit
         assert len(buffer) == 200
 

src/borg/testsuite/repository.py

@@ -717,6 +717,8 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
 
         assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve']
         args = MockArgs()
+        # XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
+        logging.getLogger().setLevel(logging.INFO)
         # note: test logger is on info log level, so --info gets added automagically
         assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info']
         args.remote_path = 'borg-0.28.2'

src/borg/xattr.py

@@ -111,7 +111,7 @@ def split_lstring(buf):
 
 
 class BufferTooSmallError(Exception):
-    """the buffer given to an xattr function was too small for the result"""
+    """the buffer given to an xattr function was too small for the result."""
 
 
 def _check(rv, path=None, detect_buffer_too_small=False):
@@ -202,7 +202,7 @@ if sys.platform.startswith('linux'):  # pragma: linux only
 
         n, buf = _listxattr_inner(func, path)
         return [os.fsdecode(name) for name in split_string0(buf[:n])
-                if not name.startswith(b'system.posix_acl_')]
+                if name and not name.startswith(b'system.posix_acl_')]
 
     def getxattr(path, name, *, follow_symlinks=True):
         def func(path, name, buf, size):
@@ -258,7 +258,7 @@ elif sys.platform == 'darwin':  # pragma: darwin only
                     return libc.listxattr(path, buf, size, XATTR_NOFOLLOW)
 
         n, buf = _listxattr_inner(func, path)
-        return [os.fsdecode(name) for name in split_string0(buf[:n])]
+        return [os.fsdecode(name) for name in split_string0(buf[:n]) if name]
 
     def getxattr(path, name, *, follow_symlinks=True):
         def func(path, name, buf, size):
@@ -317,7 +317,7 @@ elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
                     return libc.extattr_list_link(path, ns, buf, size)
 
         n, buf = _listxattr_inner(func, path)
-        return [os.fsdecode(name) for name in split_lstring(buf[:n])]
+        return [os.fsdecode(name) for name in split_lstring(buf[:n]) if name]
 
     def getxattr(path, name, *, follow_symlinks=True):
         def func(path, name, buf, size):

tox.ini

@@ -9,7 +9,7 @@ deps =
      -rrequirements.d/development.txt
      -rrequirements.d/attic.txt
      -rrequirements.d/fuse.txt
-commands = py.test -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
+commands = py.test -n 8 -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *