Merge branch '1.0-maint' into merge-1.0-maint

# Conflicts:
#	MANIFEST.in
#	Vagrantfile
#	docs/changes.rst
#	docs/usage/mount.rst.inc
#	src/borg/archiver.py
#	src/borg/fuse.py
#	src/borg/repository.py

MANIFEST.in

@@ -1,12 +1,8 @@
 include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in
-graft src
-recursive-exclude src *.pyc
-recursive-exclude src *.pyo
-recursive-exclude src *.so
-recursive-include docs *
-recursive-exclude docs *.pyc
-recursive-exclude docs *.pyo
-prune docs/_build
+exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
 prune .travis
 prune .github
-exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
+graft src
+graft docs
+prune docs/_build
+global-exclude *.py[co] *.orig *.so *.dll

Vagrantfile

@@ -63,11 +63,13 @@ end
 def packages_darwin
   return <<-EOF
     # install all the (security and other) updates
+    sudo softwareupdate --ignore iTunesX
+    sudo softwareupdate --ignore iTunes
     sudo softwareupdate --install --all
     # get osxfuse 3.x release code from github:
-    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.3/osxfuse-3.5.3.dmg >osxfuse.dmg
+    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.4/osxfuse-3.5.4.dmg >osxfuse.dmg
     MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
-    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.3.pkg" -target /
+    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.4.pkg" -target /
     sudo chown -R vagrant /usr/local  # brew must be able to create stuff here
     ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     brew update
@@ -83,11 +85,13 @@ end
 
 def packages_freebsd
   return <<-EOF
+    # VM has no hostname set
+    hostname freebsd
     # install all the (security and other) updates, base system
     freebsd-update --not-running-from-cron fetch install
     # for building borgbackup and dependencies:
     pkg install -y openssl liblz4 fusefs-libs pkgconf
-    pkg install -y fakeroot git bash
+    pkg install -y git bash
     # for building python:
     pkg install -y sqlite3
     # make bash default / work:
@@ -227,7 +231,7 @@ def install_pythons(boxname)
     pyenv install 3.4.0  # tests
     pyenv install 3.5.0  # tests
     pyenv install 3.6.0  # tests
-    pyenv install 3.5.2  # binary build, use latest 3.5.x release
+    pyenv install 3.5.3  # binary build, use latest 3.5.x release
     pyenv rehash
   EOF
 end
@@ -245,8 +249,8 @@ def build_pyenv_venv(boxname)
     . ~/.bash_profile
     cd /vagrant/borg
     # use the latest 3.5 release
-    pyenv global 3.5.2
-    pyenv virtualenv 3.5.2 borg-env
+    pyenv global 3.5.3
+    pyenv virtualenv 3.5.3 borg-env
     ln -s ~/.pyenv/versions/borg-env .
   EOF
 end
@@ -446,6 +450,16 @@ Vagrant.configure(2) do |config|
   # OS X
   config.vm.define "darwin64" do |b|
     b.vm.box = "jhcook/yosemite-clitools"
+    b.vm.provider :virtualbox do |v|
+      v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64']
+      v.customize ['modifyvm', :id, '--paravirtprovider', 'default']
+      # Adjust CPU settings according to
+      # https://github.com/geerlingguy/macos-virtualbox-vm
+      v.customize ['modifyvm', :id, '--cpuidset',
+                   '00000001', '000306a9', '00020800', '80000201', '178bfbff']
+      # Disable USB variant requiring Virtualbox proprietary extension pack
+      v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off']
+    end
     b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64")
     b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
@@ -458,11 +472,11 @@ Vagrant.configure(2) do |config|
   end
 
   # BSD
-  # note: the FreeBSD-10.3-STABLE box needs "vagrant up" twice to start.
+  # note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start.
   config.vm.define "freebsd64" do |b|
-    b.vm.box = "freebsd/FreeBSD-10.3-STABLE"
+    b.vm.box = "freebsd/FreeBSD-10.3-RELEASE"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.ssh.shell = "sh"
     b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd

docs/changes.rst

@@ -5,8 +5,8 @@ This section is used for infos about security and corruption issues.
 
 .. _tam_vuln:
 
-Pre-1.0.9 manifest spoofing vulnerability
------------------------------------------
+Pre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099)
+----------------------------------------------------------
 
 A flaw in the cryptographic authentication scheme in Borg allowed an attacker
 to spoof the manifest. The attack requires an attacker to be able to
@@ -54,7 +54,9 @@ Vulnerability time line:
 
 * 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore)
 * 2016-11-20: First patch
-* 2016-12-18: Released fixed versions: 1.0.9, 1.1.0b3
+* 2016-12-20: Released fixed version 1.0.9
+* 2017-01-02: CVE was assigned
+* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source)
 
 .. _attic013_check_corruption:
 
@@ -207,8 +209,8 @@ Other changes:
   - remove all BORG_* env vars from the outer environment
 
 
-Version 1.0.10rc1 (not released yet)
-------------------------------------
+Version 1.0.10rc1 (2017-01-29)
+------------------------------
 
 Bug fixes:
 
@@ -223,9 +225,16 @@ Bug fixes:
 - Fixed change-passphrase crashing with unencrypted repositories, #1978
 - Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997
 - borg check: print non-exit-code warning if --last or --prefix aren't fulfilled
+- fix bad parsing of wrong repo location syntax
+- create: don't create hard link refs to failed files,
+  mount: handle invalid hard link refs, #2092
+- detect mingw byte order, #2073
+- creating a new segment: use "xb" mode, #2099
+- mount: umount on SIGINT/^C when in foreground, #2082
 
 Other changes:
 
+- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002
 - xattr: ignore empty names returned by llistxattr(2) et al
 - Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT,
   SIGBUS and SIGILL signals to dump the Python traceback.
@@ -235,8 +244,11 @@ Other changes:
 - tests:
 
   - vagrant / travis / tox: add Python 3.6 based testing
-  - vagrant: fix openbsd repo, fixes #2042
-  - vagrant: fix the freebsd64 machine, #2037
+  - vagrant: fix openbsd repo, #2042
+  - vagrant: fix the freebsd64 machine, #2037 #2067
+  - vagrant: use python 3.5.3 to build binaries, #2078
+  - vagrant: use osxfuse 3.5.4 for tests / to build binaries
+    vagrant: improve darwin64 VM settings
   - travis: fix osxfuse install (fixes OS X testing on Travis CI)
   - travis: require succeeding OS X tests, #2028
   - travis: use latest pythons for OS X based testing
@@ -248,12 +260,18 @@ Other changes:
   - language clarification - VM backup FAQ
   - borg create: document how to backup stdin, #2013
   - borg upgrade: fix incorrect title levels
+  - add CVE numbers for issues fixed in 1.0.9, #2106
 - fix typos (taken from Debian package patch)
 - remote: include data hexdump in "unexpected RPC data" error message
 - remote: log SSH command line at debug level
 - API_VERSION: use numberspaces, #2023
 - remove .github from pypi package, #2051
 - add pip and setuptools to requirements file, #2030
+- SyncFile: fix use of fd object after close (cosmetic)
+- Manifest.in: simplify, exclude *.{so,dll,orig}, #2066
+- ignore posix_fadvise errors in repository.py, #2095
+  (works around issues with docker on ARM)
+- make LoggedIO.close_segment reentrant, avoid reentrance
 
 
 Version 1.0.9 (2016-12-20)
@@ -264,10 +282,14 @@ Security fixes:
 - A flaw in the cryptographic authentication scheme in Borg allowed an attacker
   to spoof the manifest. See :ref:`tam_vuln` above for the steps you should
   take.
+
+  CVE-2016-10099 was assigned to this vulnerability.
 - borg check: When rebuilding the manifest (which should only be needed very rarely)
   duplicate archive names would be handled on a "first come first serve" basis, allowing
   an attacker to apparently replace archives.
 
+  CVE-2016-10100 was assigned to this vulnerability.
+
 Bug fixes:
 
 - borg check:

src/borg/_hashindex.c

@@ -12,16 +12,24 @@
 #include <sys/isa_defs.h>
 #endif
 
-#if (defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN)) ||  \
-    (defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun))
+#if (defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)) ||  \
+    (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) || \
+    (defined(_BIG_ENDIAN) && defined(__SVR4)&&defined(__sun))
+#define BORG_BIG_ENDIAN 1
+#elif (defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)) || \
+      (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \
+      (defined(_LITTLE_ENDIAN) && defined(__SVR4)&&defined(__sun))
+#define BORG_BIG_ENDIAN 0
+#else
+#error Unknown byte order
+#endif
+
+#if BORG_BIG_ENDIAN
 #define _le32toh(x) __builtin_bswap32(x)
 #define _htole32(x) __builtin_bswap32(x)
-#elif (defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN)) || \
-      (defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun))
+#else
 #define _le32toh(x) (x)
 #define _htole32(x) (x)
-#else
-#error Unknown byte order
 #endif
 
 #define MAGIC "BORG_IDX"

src/borg/archive.py

@@ -838,8 +838,6 @@ Number of files: {0.stats.nfiles}'''.format(
                 self.add_item(item)
                 status = 'h'  # regular file, hardlink (to already seen inodes)
                 return status
-            else:
-                self.hard_links[st.st_ino, st.st_dev] = safe_path
         is_special_file = is_special(st.st_mode)
         if not is_special_file:
             path_hash = self.key.id_hash(safe_encode(os.path.join(self.cwd, path)))
@@ -890,6 +888,9 @@ Number of files: {0.stats.nfiles}'''.format(
             item.mode = stat.S_IFREG | stat.S_IMODE(item.mode)
         self.stats.nfiles += 1
         self.add_item(item)
+        if st.st_nlink > 1 and source is None:
+            # Add the hard link reference *after* the file has been added to the archive.
+            self.hard_links[st.st_ino, st.st_dev] = safe_path
         return status
 
     @staticmethod

src/borg/archiver.py

@@ -2244,6 +2244,13 @@ class Archiver:
         to tweak the performance. It sets the number of cached data chunks; additional
         memory usage can be up to ~8 MiB times this number. The default is the number
         of CPU cores.
+
+        When the daemonized process receives a signal or crashes, it does not unmount.
+        Unmounting in these cases could cause an active rsync or similar process
+        to unintentionally delete data.
+
+        When running in the foreground ^C/SIGINT unmounts cleanly, but other
+        signals or crashes do not.
         """)
         subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False,
                                           description=self.do_mount.__doc__,

src/borg/fuse.py

@@ -5,6 +5,7 @@ import stat
 import tempfile
 import time
 from collections import defaultdict
+from signal import SIGINT
 from distutils.version import LooseVersion
 from zlib import adler32
 
@@ -125,7 +126,8 @@ class FuseOperations(llfuse.Operations):
         umount = False
         try:
             signal = fuse_main()
-            umount = (signal is None)  # no crash and no signal -> umount request
+            # no crash and no signal (or it's ^C and we're in the foreground) -> umount request
+            umount = (signal is None or (signal == SIGINT and foreground))
         finally:
             llfuse.close(umount)
 
@@ -190,6 +192,7 @@ class FuseOperations(llfuse.Operations):
                 path = os.fsencode(os.path.normpath(item.path))
                 self.file_versions[path] = version
 
+        path = item.path
         del item.path  # safe some space
         if 'source' in item and stat.S_ISREG(item.mode):
             # a hardlink, no contents, <source> is the hardlink master
@@ -199,7 +202,11 @@ class FuseOperations(llfuse.Operations):
                 version = self.file_versions[source]
                 source = make_versioned_name(source, version, add_dir=True)
                 name = make_versioned_name(name, version)
-            inode = self._find_inode(source, prefix)
+            try:
+                inode = self._find_inode(source, prefix)
+            except KeyError:
+                logger.warning('Skipping broken hard link: %s -> %s', path, item.source)
+                return
             item = self.cache.get(inode)
             item.nlink = item.get('nlink', 1) + 1
             self.items[inode] = item

src/borg/helpers.py

@@ -926,11 +926,17 @@ class Location:
     """
 
     # path must not contain :: (it ends at :: or string end), but may contain single colons.
-    # to avoid ambiguities with other regexes, it must also not start with ":".
+    # to avoid ambiguities with other regexes, it must also not start with ":" nor with "//" nor with "ssh://".
     path_re = r"""
-        (?!:)                                               # not starting with ":"
+        (?!(:|//|ssh://))                                   # not starting with ":" or // or ssh://
         (?P<path>([^:]|(:(?!:)))+)                          # any chars, but no "::"
         """
+    # abs_path must not contain :: (it ends at :: or string end), but may contain single colons.
+    # it must start with a / and that slash is part of the path.
+    abs_path_re = r"""
+        (?P<path>(/([^:]|(:(?!:)))+))                       # start with /, then any chars, but no "::"
+        """
+
     # optional ::archive_name at the end, archive name must not contain "/".
     # borg mount's FUSE filesystem creates one level of directories from
     # the archive names and of course "/" is not valid in a directory name.
@@ -945,7 +951,7 @@ class Location:
         (?P<proto>ssh)://                                   # ssh://
         """ + optional_user_re + r"""                       # user@  (optional)
         (?P<host>[^:/]+)(?::(?P<port>\d+))?                 # host or host:port
-        """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive
+        """ + abs_path_re + optional_archive_re, re.VERBOSE)  # path or path::archive
 
     file_re = re.compile(r"""
         (?P<proto>file)://                                  # file://

src/borg/platform/__init__.py

@@ -8,7 +8,7 @@ Public APIs are documented in platform.base.
 
 from .base import acl_get, acl_set
 from .base import set_flags, get_flags
-from .base import SaveFile, SyncFile, sync_dir, fdatasync
+from .base import SaveFile, SyncFile, sync_dir, fdatasync, safe_fadvise
 from .base import swidth, umount, API_VERSION
 from .base import process_alive, get_process_id, local_pid_alive
 

src/borg/platform/base.py

@@ -63,6 +63,21 @@ def sync_dir(path):
         os.close(fd)
 
 
+def safe_fadvise(fd, offset, len, advice):
+    if hasattr(os, 'posix_fadvise'):
+        try:
+            os.posix_fadvise(fd, offset, len, advice)
+        except OSError:
+            # usually, posix_fadvise can't fail for us, but there seem to
+            # be failures when running borg under docker on ARM, likely due
+            # to a bug outside of borg.
+            # also, there is a python wrapper bug, always giving errno = 0.
+            # https://github.com/borgbackup/borg/issues/2095
+            # as this call is not critical for correct function (just to
+            # optimize cache usage), we ignore these errors.
+            pass
+
+
 class SyncFile:
     """
     A file class that is supposed to enable write ordering (one way or another) and data durability after close().
@@ -103,15 +118,21 @@ class SyncFile:
         from .. import platform
         self.fd.flush()
         platform.fdatasync(self.fileno)
-        if hasattr(os, 'posix_fadvise'):
-            os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
+        # tell the OS that it does not need to cache what we just wrote,
+        # avoids spoiling the cache for the OS and other processes.
+        safe_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
 
     def close(self):
         """sync() and close."""
         from .. import platform
-        self.sync()
-        self.fd.close()
-        platform.sync_dir(os.path.dirname(self.fd.name))
+        dirname = None
+        try:
+            dirname = os.path.dirname(self.fd.name)
+            self.sync()
+        finally:
+            self.fd.close()
+            if dirname:
+                platform.sync_dir(dirname)
 
 
 class SaveFile:

src/borg/platform/linux.pyx

@@ -8,6 +8,7 @@ from ..helpers import posix_acl_use_stored_uid_gid
 from ..helpers import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
 from .base import SyncFile as BaseSyncFile
+from .base import safe_fadvise
 from .posix import swidth
 
 from libc cimport errno
@@ -216,7 +217,7 @@ cdef _sync_file_range(fd, offset, length, flags):
     assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
     if sync_file_range(fd, offset, length, flags) != 0:
         raise OSError(errno.errno, os.strerror(errno.errno))
-    os.posix_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED)
+    safe_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED)
 
 cdef unsigned PAGE_MASK = resource.getpagesize() - 1
 
@@ -251,7 +252,9 @@ class SyncFile(BaseSyncFile):
     def sync(self):
         self.fd.flush()
         os.fdatasync(self.fileno)
-        os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
+        # tell the OS that it does not need to cache what we just wrote,
+        # avoids spoiling the cache for the OS and other processes.
+        safe_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
 
 
 def umount(mountpoint):

src/borg/repository.py

@@ -21,7 +21,7 @@ from .helpers import yes
 from .locking import Lock, LockError, LockErrorT
 from .logger import create_logger
 from .lrucache import LRUCache
-from .platform import SaveFile, SyncFile, sync_dir
+from .platform import SaveFile, SyncFile, sync_dir, safe_fadvise
 from .crc32 import crc32
 
 logger = create_logger(__name__)
@@ -909,8 +909,7 @@ class LoggedIO:
         self.fds = None  # Just to make sure we're disabled
 
     def close_fd(self, fd):
-        if hasattr(os, 'posix_fadvise'):  # only on UNIX
-            os.posix_fadvise(fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
+        safe_fadvise(fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
         fd.close()
 
     def segment_iterator(self, segment=None, reverse=False):
@@ -1017,11 +1016,12 @@ class LoggedIO:
             return fd
 
     def close_segment(self):
-        if self._write_fd:
+        # set self._write_fd to None early to guard against reentry from error handling code pathes:
+        fd, self._write_fd = self._write_fd, None
+        if fd is not None:
             self.segment += 1
             self.offset = 0
-            self._write_fd.close()
-            self._write_fd = None
+            fd.close()
 
     def delete_segment(self, segment):
         if segment in self.fds:

src/borg/testsuite/helpers.py

@@ -137,6 +137,11 @@ class TestLocationWithoutEnv:
         location_time2 = Location('/some/path::archive{now:%s}')
         assert location_time1.archive != location_time2.archive
 
+    def test_bad_syntax(self):
+        with pytest.raises(ValueError):
+            # this is invalid due to the 2nd colon, correct: 'ssh://user@host/path'
+            Location('ssh://user@host:/path')
+
 
 class TestLocationWithEnv:
     def test_ssh(self, monkeypatch):