Merge branch '1.0-maint'

Vagrantfile

@@ -25,6 +25,8 @@ def packages_debianoid
     # for building borgbackup and dependencies:
     apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
     usermod -a -G fuse $username
+    chgrp fuse /dev/fuse
+    chmod 666 /dev/fuse
     apt-get install -y fakeroot build-essential git
     apt-get install -y python3-dev python3-setuptools
     # for building python:
@@ -45,6 +47,8 @@ def packages_redhatted
     # for building borgbackup and dependencies:
     yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig
     usermod -a -G fuse vagrant
+    chgrp fuse /dev/fuse
+    chmod 666 /dev/fuse
     yum install -y fakeroot gcc git patch
     # needed to compile msgpack-python (otherwise it will use slow fallback code):
     yum install -y gcc-c++
@@ -96,6 +100,8 @@ def packages_freebsd
     kldload fuse
     sysctl vfs.usermount=1
     pw groupmod operator -M vagrant
+    # /dev/fuse has group operator
+    chmod 666 /dev/fuse
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
     # install all the (security and other) updates, packages
     pkg update
@@ -106,10 +112,6 @@ end
 def packages_openbsd
   return <<-EOF
     . ~/.profile
-    mkdir -p /home/vagrant/borg
-    rsync -aH /vagrant/borg/ /home/vagrant/borg/
-    rm -rf /vagrant/borg
-    ln -sf /home/vagrant/borg /vagrant/
     pkg_add bash
     chsh -s /usr/local/bin/bash vagrant
     pkg_add openssl
@@ -121,6 +123,8 @@ def packages_openbsd
     easy_install-3.4 pip
     pip3 install virtualenv
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+    # avoid that breaking llfuse install breaks borgbackup install under tox:
+    sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
   EOF
 end
 
@@ -146,6 +150,8 @@ def packages_netbsd
     easy_install-3.4 pip
     pip install virtualenv
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+    # fuse does not work good enough (see above), do not install llfuse:
+    sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
   EOF
 end
 
@@ -273,13 +279,13 @@ def run_tests(boxname)
     . ~/.bash_profile
     cd /vagrant/borg/borg
     . ../borg-env/bin/activate
-    if which pyenv > /dev/null; then
+    if which pyenv 2> /dev/null; then
       # for testing, use the earliest point releases of the supported python versions:
       pyenv global 3.4.0 3.5.0
       pyenv local 3.4.0 3.5.0
     fi
     # otherwise: just use the system python
-    if which fakeroot > /dev/null; then
+    if which fakeroot 2> /dev/null; then
       echo "Running tox WITH fakeroot -u"
       fakeroot -u tox --skip-missing-interpreters
     else
@@ -304,7 +310,7 @@ end
 
 Vagrant.configure(2) do |config|
   # use rsync to copy content to the folder
-  config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"]
+  config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false
   # do not let the VM access . on the host machine via the default shared folder!
   config.vm.synced_folder ".", "/vagrant", disabled: true
 
@@ -443,7 +449,7 @@ Vagrant.configure(2) do |config|
   end
 
   config.vm.define "openbsd64" do |b|
-    b.vm.box = "kaorimatz/openbsd-5.9-amd64"
+    b.vm.box = "openbsd60-64"  # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed
     b.vm.provider :virtualbox do |v|
       v.memory = 768
     end
@@ -454,7 +460,7 @@ Vagrant.configure(2) do |config|
   end
 
   config.vm.define "netbsd64" do |b|
-    b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64"
+    b.vm.box = "netbsd70-64"
     b.vm.provider :virtualbox do |v|
       v.memory = 768
     end

docs/api.rst

@@ -26,6 +26,10 @@ API Documentation
     :members:
     :undoc-members:
 
+.. automodule:: borg.keymanager
+    :members:
+    :undoc-members:
+
 .. automodule:: borg.nonces
     :members:
     :undoc-members:

docs/changes.rst

@@ -218,8 +218,8 @@ Other changes:
   - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945
 
 
-Version 1.0.8rc1 (not released yet)
------------------------------------
+Version 1.0.8rc1 (2016-10-17)
+-----------------------------
 
 Bug fixes:
 
@@ -231,15 +231,22 @@ Bug fixes:
   also correctly processes broken symlinks. before this regressed to a crash
   (5b45385) a broken symlink would've been skipped.
 - process_symlink: fix missing backup_io()
-  Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting dirents
-  and dispatching to process_symlink.
-- yes(): abort on wrong answers, saying so
+  Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting
+  dirents and dispatching to process_symlink.
+- yes(): abort on wrong answers, saying so, #1622
 - fixed exception borg serve raised when connection was closed before reposiory
   was openend. add an error message for this.
 - fix read-from-closed-FD issue, #1551
   (this seems not to get triggered in 1.0.x, but was discovered in master)
 - hashindex: fix iterators (always raise StopIteration when exhausted)
   (this seems not to get triggered in 1.0.x, but was discovered in master)
+- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655
+- allow repo pathes with colons, fixes #1705
+- update changed repo location immediately after acceptance, #1524
+- fix debug get-obj / delete-obj crash if object not found and remote repo,
+  #1684
+- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib
+  on Mac OS X (avoids mismatch lib <-> driver), #1619
 
 New features:
 
@@ -250,6 +257,8 @@ New features:
   special "paper" format with by line checksums for printed backups. For the
   paper format, the import is an interactive process which checks each line as
   soon as it is input.
+- add "borg debug-refcount-obj" to determine a repo objects' referrer counts,
+  #1352
 
 Other changes:
 
@@ -258,10 +267,19 @@ Other changes:
 - setup.py: Add subcommand support to build_usage.
 - remote: change exception message for unexpected RPC data format to indicate
   dataflow direction.
-- vagrant:
+- improved messages / error reporting:
+
+  - IntegrityError: add placeholder for message, so that the message we give
+    appears not only in the traceback, but also in the (short) error message,
+    #1572
+  - borg.key: include chunk id in exception msgs, #1571
+  - better messages for cache newer than repo, fixes #1700
+- vagrant (testing/build VMs):
 
   - upgrade OSXfuse / FUSE for macOS to 3.5.2
-  - update Debian Wheezy boxes to 7.11
+  - update Debian Wheezy boxes, #1686
+  - openbsd / netbsd: use own boxes, fixes misc rsync installation and
+    fuse/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728
 - docs:
 
   - add docs for "key export" and "key import" commands, #1641
@@ -277,12 +295,17 @@ Other changes:
   - add debug-info usage help file
   - internals.rst: fix typos
   - setup.py: fix build_usage to always process all commands
+  - added docs explaining multiple --restrict-to-path flags, #1602
+  - add more specific warning about write-access debug commands, #1587
+  - clarify FAQ regarding backup of virtual machines, #1672
 - tests:
 
   - work around fuse xattr test issue with recent fakeroot
   - simplify repo/hashindex tests
   - travis: test fuse-enabled borg, use trusty to have a recent FUSE
   - re-enable fuse tests for RemoteArchiver (no deadlocks any more)
+  - clean env for pytest based tests, #1714
+  - fuse_mount contextmanager: accept any options
 
 
 Version 1.0.7 (2016-08-19)

docs/deployment.rst

@@ -149,10 +149,10 @@ package manager to install and keep borg up-to-date.
     - authorized_key: user="{{ user }}"
                       key="{{ item.key }}"
                       key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc'
-      with_items: auth_users
+      with_items: "{{ auth_users }}"
     - file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file
     - file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory
-      with_items: auth_users
+      with_items: "{{ auth_users }}"
 
 Salt
 ----

docs/usage.rst

@@ -897,9 +897,14 @@ That's all to it.
 Drawbacks
 +++++++++
 
-As data is only appended, and nothing deleted, commands like ``prune`` or ``delete``
+As data is only appended, and nothing removed, commands like ``prune`` or ``delete``
 won't free disk space, they merely tag data as deleted in a new transaction.
 
+Be aware that as soon as you write to the repo in non-append-only mode (e.g. prune,
+delete or create archives from an admin machine), it will remove the deleted objects
+permanently (including the ones that were already marked as deleted, but not removed,
+in append-only mode).
+
 Note that you can go back-and-forth between normal and append-only operation by editing
 the configuration file, it's not a "one way trip".
 

requirements.d/fuse.txt

@@ -1,4 +1,4 @@
 # low-level FUSE support library for "borg mount"
-# see comments setup.py about this version requirement.
+# please see the comments in setup.py about llfuse.
 llfuse<2.0
 

setup.py

@@ -23,12 +23,17 @@ on_rtd = os.environ.get('READTHEDOCS')
 # Also, we might use some rather recent API features.
 install_requires = ['msgpack-python>=0.4.6', ]
 
+# note for package maintainers: if you package borgbackup for distribution,
+# please add llfuse as a *requirement* on all platforms that have a working
+# llfuse package. "borg mount" needs llfuse to work.
+# if you do not have llfuse, do not require it, most of borgbackup will work.
 extras_require = {
     # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
     # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
+    # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 2.0 will break API
     'fuse': ['llfuse<2.0', ],
 }

src/borg/__init__.py

@@ -1,3 +1,6 @@
-# This is a python package
+from distutils.version import LooseVersion
 
 from ._version import version as __version__
+
+
+__version_tuple__ = tuple(LooseVersion(__version__).version[:3])

src/borg/archiver.py

@@ -1176,7 +1176,7 @@ class Archiver:
         else:
             try:
                 data = repository.get(id)
-            except repository.ObjectNotFound:
+            except Repository.ObjectNotFound:
                 print("object %s not found." % hex_id)
             else:
                 with open(args.path, "wb") as f:
@@ -1210,13 +1210,29 @@ class Archiver:
                     repository.delete(id)
                     modified = True
                     print("object %s deleted." % hex_id)
-                except repository.ObjectNotFound:
+                except Repository.ObjectNotFound:
                     print("object %s not found." % hex_id)
         if modified:
             repository.commit()
         print('Done.')
         return EXIT_SUCCESS
 
+    @with_repository(manifest=False, exclusive=True, cache=True)
+    def do_debug_refcount_obj(self, args, repository, manifest, key, cache):
+        """display refcounts for the objects with the given IDs"""
+        for hex_id in args.ids:
+            try:
+                id = unhexlify(hex_id)
+            except ValueError:
+                print("object id %s is invalid." % hex_id)
+            else:
+                try:
+                    refcount = cache.chunks[id][0]
+                    print("object %s has %d referrers [info from chunks cache]." % (hex_id, refcount))
+                except KeyError:
+                    print("object %s not found [info from chunks cache]." % hex_id)
+        return EXIT_SUCCESS
+
     @with_repository(lock=False, manifest=False)
     def do_break_lock(self, args, repository):
         """Break the repository lock (e.g. in case it was left by a dead borg."""
@@ -1344,7 +1360,19 @@ class Archiver:
 
         {borgversion}
 
-            The version of borg.
+            The version of borg, e.g.: 1.0.8rc1
+
+        {borgmajor}
+
+            The version of borg, only the major version, e.g.: 1
+
+        {borgminor}
+
+            The version of borg, only major and minor version, e.g.: 1.0
+
+        {borgpatch}
+
+            The version of borg, only major, minor and patch version, e.g.: 1.0.8
 
         Examples::
 
@@ -1777,8 +1805,8 @@ class Archiver:
         '.checkpoint.N' (with N being a number), because these names are used for
         checkpoints and treated in special ways.
 
-        In the archive name, you may use the following format tags:
-        {now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}, {uuid4}, {borgversion}
+        In the archive name, you may use the following placeholders:
+        {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others.
 
         To speed up pulling backups over sshfs and similar network file systems which do
         not provide correct inode information the --ignore-inode flag can be used. This
@@ -2541,6 +2569,21 @@ class Archiver:
         subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
                                help='hex object ID(s) to delete from the repo')
 
+        debug_refcount_obj_epilog = textwrap.dedent("""
+        This command displays the reference count for objects from the repository.
+        """)
+        subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False,
+                                          description=self.do_debug_refcount_obj.__doc__,
+                                          epilog=debug_refcount_obj_epilog,
+                                          formatter_class=argparse.RawDescriptionHelpFormatter,
+                                          help='show refcount for object from repository (debug)')
+        subparser.set_defaults(func=self.do_debug_refcount_obj)
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False),
+                               help='repository to use')
+        subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
+                               help='hex object ID(s) to show refcounts for')
+
         return parser
 
     @staticmethod

src/borg/helpers.py

@@ -34,6 +34,7 @@ from .logger import create_logger
 logger = create_logger()
 
 from . import __version__ as borg_version
+from . import __version_tuple__ as borg_version_tuple
 from . import chunker
 from . import crypto
 from . import hashindex
@@ -664,6 +665,9 @@ def replace_placeholders(text):
         'user': uid2user(os.getuid(), os.getuid()),
         'uuid4': str(uuid.uuid4()),
         'borgversion': borg_version,
+        'borgmajor': '%d' % borg_version_tuple[:1],
+        'borgminor': '%d.%d' % borg_version_tuple[:2],
+        'borgpatch': '%d.%d.%d' % borg_version_tuple[:3],
     }
     return format_line(text, data)
 
@@ -945,26 +949,32 @@ class Location:
         return True
 
     def _parse(self, text):
+        def normpath_special(p):
+            # avoid that normpath strips away our relative path hack and even makes p absolute
+            relative = p.startswith('/./')
+            p = os.path.normpath(p)
+            return ('/.' + p) if relative else p
+
         m = self.ssh_re.match(text)
         if m:
             self.proto = m.group('proto')
             self.user = m.group('user')
             self.host = m.group('host')
             self.port = m.group('port') and int(m.group('port')) or None
-            self.path = os.path.normpath(m.group('path'))
+            self.path = normpath_special(m.group('path'))
             self.archive = m.group('archive')
             return True
         m = self.file_re.match(text)
         if m:
             self.proto = m.group('proto')
-            self.path = os.path.normpath(m.group('path'))
+            self.path = normpath_special(m.group('path'))
             self.archive = m.group('archive')
             return True
         m = self.scp_re.match(text)
         if m:
             self.user = m.group('user')
             self.host = m.group('host')
-            self.path = os.path.normpath(m.group('path'))
+            self.path = normpath_special(m.group('path'))
             self.archive = m.group('archive')
             self.proto = self.host and 'ssh' or 'file'
             return True
@@ -995,9 +1005,9 @@ class Location:
             return self.path
         else:
             if self.path and self.path.startswith('~'):
-                path = '/' + self.path
+                path = '/' + self.path  # /~/x = path x relative to home dir
             elif self.path and not self.path.startswith('/'):
-                path = '/~/' + self.path
+                path = '/./' + self.path  # /./x = path x relative to cwd
             else:
                 path = self.path
             return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '',

src/borg/remote.py

@@ -149,8 +149,10 @@ class RepositoryServer:  # pragma: no cover
 
     def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False):
         path = os.fsdecode(path)
-        if path.startswith('/~'):
-            path = os.path.join(get_home_dir(), path[2:])
+        if path.startswith('/~'):  # /~/x = path x relative to home dir, /~username/x = relative to "user" home dir
+            path = os.path.join(get_home_dir(), path[2:])  # XXX check this (see also 1.0-maint), is it correct for ~u?
+        elif path.startswith('/./'):  # /./x = path x relative to cwd
+            path = path[3:]
         path = os.path.realpath(path)
         if self.restrict_to_paths:
             # if --restrict-to-path P is given, we make sure that we only operate in/below path P.

src/borg/testsuite/__init__.py

@@ -117,6 +117,15 @@ def is_utime_fully_supported():
         return False
 
 
+def no_selinux(x):
+    # selinux fails our FUSE tests, thus ignore selinux xattrs
+    SELINUX_KEY = 'security.selinux'
+    if isinstance(x, dict):
+        return {k: v for k, v in x.items() if k != SELINUX_KEY}
+    if isinstance(x, list):
+        return [k for k in x if k != SELINUX_KEY]
+
+
 class BaseTestCase(unittest.TestCase):
     """
     """
@@ -176,8 +185,8 @@ class BaseTestCase(unittest.TestCase):
                 else:
                     d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
                     d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
-            d1.append(get_all(path1, follow_symlinks=False))
-            d2.append(get_all(path2, follow_symlinks=False))
+            d1.append(no_selinux(get_all(path1, follow_symlinks=False)))
+            d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
             self.assert_equal(d1, d2)
         for sub_diff in diff.subdirs.values():
             self._assert_dirs_equal_cmp(sub_diff)

src/borg/testsuite/archiver.py

@@ -39,8 +39,10 @@ from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse
-from . import BaseTestCase, changedir, environment_variable
+from . import BaseTestCase, changedir, environment_variable, no_selinux
 from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported
+from .platform import fakeroot_detected
+
 
 src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
@@ -1428,7 +1430,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             in_fn = 'input/fusexattr'
             out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
             if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
-                assert xattr.listxattr(out_fn) == ['user.foo', ]
+                assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ]
                 assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
             else:
                 assert xattr.listxattr(out_fn) == []
@@ -1988,6 +1990,12 @@ class ArchiverTestCaseBinary(ArchiverTestCase):
     def test_overwrite(self):
         pass
 
+    def test_fuse(self):
+        if fakeroot_detected():
+            unittest.skip('test_fuse with the binary is not compatible with fakeroot')
+        else:
+            super().test_fuse()
+
 
 class ArchiverCheckTestCase(ArchiverTestCaseBase):
 

src/borg/testsuite/helpers.py

@@ -84,6 +84,8 @@ class TestLocationWithoutEnv:
             "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')"
         assert repr(Location('/some/absolute/path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)"
+        assert repr(Location('ssh://user@host/some/path')) == \
+               "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
 
     def test_relpath(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
@@ -91,6 +93,12 @@ class TestLocationWithoutEnv:
             "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')"
         assert repr(Location('some/relative/path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
+        assert repr(Location('ssh://user@host/./some/path')) == \
+               "Location(proto='ssh', user='user', host='host', port=None, path='/./some/path', archive=None)"
+        assert repr(Location('ssh://user@host/~/some/path')) == \
+               "Location(proto='ssh', user='user', host='host', port=None, path='/~/some/path', archive=None)"
+        assert repr(Location('ssh://user@host/~user/some/path')) == \
+               "Location(proto='ssh', user='user', host='host', port=None, path='/~user/some/path', archive=None)"
 
     def test_with_colons(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
@@ -122,7 +130,7 @@ class TestLocationWithoutEnv:
                      'ssh://user@host:1234/some/path::archive']
         for location in locations:
             assert Location(location).canonical_path() == \
-                Location(Location(location).canonical_path()).canonical_path()
+                Location(Location(location).canonical_path()).canonical_path(), "failed: %s" % location
 
     def test_format_path(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)