Merge pull request #1389 from enkore/merge/1.0-maint3

Merge branch '1.0-maint' into master

README.rst

@@ -33,14 +33,14 @@ Main features
 
   Compared to other deduplication approaches, this method does NOT depend on:
 
-  * file/directory names staying the same: So you can move your stuff around 
+  * file/directory names staying the same: So you can move your stuff around
     without killing the deduplication, even between machines sharing a repo.
 
-  * complete files or time stamps staying the same: If a big file changes a 
-    little, only a few new chunks need to be stored - this is great for VMs or 
+  * complete files or time stamps staying the same: If a big file changes a
+    little, only a few new chunks need to be stored - this is great for VMs or
     raw disks.
 
-  * The absolute position of a data chunk inside a file: Stuff may get shifted 
+  * The absolute position of a data chunk inside a file: Stuff may get shifted
     and will still be found by the deduplication algorithm.
 
 **Speed**

Vagrantfile

@@ -12,12 +12,19 @@ end
 
 def packages_debianoid
   return <<-EOF
+    if id "vagrant" >/dev/null 2>&1; then
+      username='vagrant'
+      home_dir=/home/vagrant
+    else
+      username='ubuntu'
+      home_dir=/home/ubuntu
+    fi
     apt-get update
     # install all the (security and other) updates
     apt-get dist-upgrade -y
     # for building borgbackup and dependencies:
     apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
-    usermod -a -G fuse vagrant
+    usermod -a -G fuse $username
     apt-get install -y fakeroot build-essential git
     apt-get install -y python3-dev python3-setuptools
     # for building python:
@@ -27,7 +34,7 @@ def packages_debianoid
     # newer versions are not compatible with py 3.2 any more.
     easy_install3 'pip<8.0'
     pip3 install 'virtualenv<14.0'
-    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+    touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile
   EOF
 end
 
@@ -283,7 +290,13 @@ end
 def fix_perms
   return <<-EOF
     # . ~/.profile
-    chown -R vagrant /vagrant/borg
+
+    if id "vagrant" >/dev/null 2>&1; then
+      chown -R vagrant /vagrant/borg
+    else
+      chown -R ubuntu /vagrant/borg
+    fi
+
   EOF
 end
 
@@ -344,9 +357,9 @@ Vagrant.configure(2) do |config|
       v.memory = 768
     end
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("xenial64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64")
   end
 
   config.vm.define "trusty64" do |b|

docs/changes.rst

@@ -457,7 +457,7 @@ Compatibility notes:
 - disambiguate -p option, #563:
 
   - -p now is same as --progress
-  - -P now is same as --prefix  
+  - -P now is same as --prefix
 - remove deprecated "borg verify",
   use "borg extract --dry-run" instead
 - cleanup environment variable semantics, #355
@@ -522,7 +522,7 @@ New features:
 - format options for location: user, pid, fqdn, hostname, now, utcnow, user
 - borg list --list-format
 - borg prune -v --list enables the keep/prune list output, #658
- 
+
 Bug fixes:
 
 - fix _open_rb noatime handling, #657
@@ -540,7 +540,7 @@ Other changes:
 - Vagrant: drop Ubuntu Precise (12.04) - does not have Python >= 3.4
 - Vagrant: use pyinstaller v3.1.1 to build binaries
 - docs:
- 
+
   - borg upgrade: add to docs that only LOCAL repos are supported
   - borg upgrade also handles borg 0.xx -> 1.0
   - use pip extras or requirements file to install llfuse

docs/development.rst

@@ -19,6 +19,16 @@ instead of 79. We do *not* use form-feed (``^L``) characters to
 separate sections either. Compliance is tested automatically when
 you run the tests.
 
+Continuous Integration
+----------------------
+
+All pull requests go through Travis-CI_, which runs the tests on Linux
+and Mac OS X as well as the flake8 style checker. Additional Unix-like platforms
+are tested on Golem_.
+
+.. _Golem: https://golem.enkore.de/view/Borg/
+.. _Travis-CI: https://travis-ci.org/borgbackup/borg
+
 Output and Logging
 ------------------
 When writing logger calls, always use correct log level (debug only for

docs/faq.rst

@@ -47,25 +47,6 @@ If you want to have 2 independent backups, it is better to do it like this:
 - client machine ---borg create---> repo1
 - client machine ---borg create---> repo2
 
-Which file types, attributes, etc. are preserved?
--------------------------------------------------
-
-    * Directories
-    * Regular files
-    * Hardlinks (considering all files in the same archive)
-    * Symlinks (stored as symlink, the symlink is not followed)
-    * Character and block device files
-    * FIFOs ("named pipes")
-    * Name
-    * Contents
-    * Timestamps in nanosecond precision: mtime, atime, ctime
-    * IDs of owning user and owning group
-    * Names of owning user and owning group (if the IDs can be resolved)
-    * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
-    * Extended Attributes (xattrs) on Linux, OS X and FreeBSD
-    * Access Control Lists (ACL_) on Linux, OS X and FreeBSD
-    * BSD flags on OS X and FreeBSD
-
 Which file types, attributes, etc. are *not* preserved?
 -------------------------------------------------------
 
@@ -337,11 +318,8 @@ I am seeing 'A' (added) status for a unchanged file!?
 
 The files cache is used to determine whether |project_name| already
 "knows" / has backed up a file and if so, to skip the file from
-chunking. It does intentionally *not* contain files that:
-
-- have >= 10 as "entry age" (|project_name| has not seen this file for a while)
-- have a modification time (mtime) same as the newest mtime in the created
-  archive
+chunking. It does intentionally *not* contain files that have a modification
+time (mtime) same as the newest mtime in the created archive.
 
 So, if you see an 'A' status for unchanged file(s), they are likely the files
 with the most recent mtime in that archive.

docs/installation.rst

@@ -118,6 +118,76 @@ the old version using the same steps as shown above.
 .. _pyinstaller: http://www.pyinstaller.org
 .. _releases: https://github.com/borgbackup/borg/releases
 
+.. _platforms:
+
+Features & platforms
+--------------------
+
+Besides regular file and directory structures, |project_name| can preserve
+
+    * Hardlinks (considering all files in the same archive)
+    * Symlinks (stored as symlink, the symlink is not followed)
+    * Special files:
+
+        * Character and block device files (restored via mknod)
+        * FIFOs ("named pipes")
+        * Special file *contents* can be backed up in ``--read-special`` mode.
+          By default the metadata to create them with mknod(2), mkfifo(2) etc. is stored.
+    * Timestamps in nanosecond precision: mtime, atime, ctime
+    * Permissions:
+
+        * IDs of owning user and owning group
+        * Names of owning user and owning group (if the IDs can be resolved)
+        * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
+
+On some platforms additional features are supported:
+
+.. Yes/No's are grouped by reason/mechanism/reference.
+
++------------------+----------+-----------+------------+
+| Platform         | ACLs     | xattr     | Flags      |
+|                  | [#acls]_ | [#xattr]_ | [#flags]_  |
++==================+==========+===========+============+
+| Linux x86        | Yes      | Yes       | No         |
++------------------+          |           |            |
+| Linux PowerPC    |          |           |            |
++------------------+          |           |            |
+| Linux ARM        |          |           |            |
++------------------+----------+-----------+------------+
+| Mac OS X         | Yes      | Yes       | Yes (all)  |
++------------------+----------+-----------+            |
+| FreeBSD          | Yes      | Yes       |            |
++------------------+----------+-----------+            |
+| OpenBSD          | n/a      | n/a       |            |
++------------------+----------+-----------+            |
+| NetBSD           | n/a      | No [2]_   |            |
++------------------+----------+-----------+------------+
+| Solaris 11       | No [3]_              | n/a        |
++------------------+                      |            |
+| OpenIndiana      |                      |            |
++------------------+----------+-----------+------------+
+| Windows (cygwin) | No [4]_  | No        | No         |
++------------------+----------+-----------+------------+
+
+Some Distributions (e.g. Debian) run additional tests after each release, these
+are not reflected here.
+
+Other Unix-like operating systems may work as well, but have not been tested at all.
+
+Note that most of the platform-dependent features also depend on the file system.
+For example, ntfs-3g on Linux isn't able to convey NTFS ACLs.
+
+
+.. [2] Feature request :issue:`1332`
+.. [3] Feature request :issue:`1337`
+.. [4] Cygwin tries to map NTFS ACLs to permissions with varying degress of success.
+
+.. [#acls] The native access control list mechanism of the OS. This normally limits access to
+    non-native ACLs. For example, NTFS ACLs aren't completely accessible on Linux with ntfs-3g.
+.. [#xattr] extended attributes; key-value pairs attached to a file, mainly used by the OS.
+    This includes resource forks on Mac OS X.
+.. [#flags] aka *BSD flags*.
+
 .. _source-install:
 
 From Source
@@ -211,7 +281,7 @@ and commands to make fuse work for using the mount command.
      echo 'vfs.usermount=1' >> /etc/sysctl.conf
      kldload fuse
      sysctl vfs.usermount=1
-    
+
 
 Cygwin
 ++++++

docs/internals.rst

@@ -371,7 +371,7 @@ repository_id
 
 enc_key
   the key used to encrypt data with AES (256 bits)
-  
+
 enc_hmac_key
   the key used to HMAC the encrypted data (256 bits)
 

docs/misc/internals-picture.txt

@@ -11,22 +11,22 @@ BorgBackup from 10.000m
     |      |                 |
     +------+-------+         |
     |      |       |         |
- /chunk\/chunk\/chunk\...   /maybe different chunks lists\ 
+ /chunk\/chunk\/chunk\...   /maybe different chunks lists\
 +-----------------------------------------------------------------+
 |item list                                                        |
 +-----------------------------------------------------------------+
-    |                                                       
-    +-------------------------------------+--------------+  
-    |                                     |              |  
-    |                                     |              |  
-+-------------+                     +-------------+      |  
-|item0        |                     |item1        |      |  
-| - owner     |                     | - owner     |      |  
-| - size      |                     | - size      |     ... 
-| - ...       |                     | - ...       |         
-| - chunks    |                     | - chunks    |         
-+----+--------+                     +-----+-------+         
-     |                                    |                 
+    |
+    +-------------------------------------+--------------+
+    |                                     |              |
+    |                                     |              |
++-------------+                     +-------------+      |
+|item0        |                     |item1        |      |
+| - owner     |                     | - owner     |      |
+| - size      |                     | - size      |     ...
+| - ...       |                     | - ...       |
+| - chunks    |                     | - chunks    |
++----+--------+                     +-----+-------+
+     |                                    |
      | +-----+----------------------------+-----------------+
      | |     |                                              |
      +-o-----o------------+                                 |

docs/misc/prune-example.txt

@@ -14,41 +14,41 @@ Calendar view
 -------------
 
                             2015
-      January               February               March          
-Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
-          1  2  3  4                     1                     1  
- 5  6  7  8  9 10 11   2  3  4  5  6  7  8   2  3  4  5  6  7  8  
-12 13 14 15 16 17 18   9 10 11 12 13 14 15   9 10 11 12 13 14 15  
-19 20 21 22 23 24 25  16 17 18 19 20 21 22  16 17 18 19 20 21 22  
-26 27 28 29 30 31     23 24 25 26 27 28     23 24 25 26 27 28 29  
-                                            30 31                 
-
-       April                  May                   June          
-Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
-       1  2  3  4  5               1  2  3   1  2  3  4  5  6  7  
- 6  7  8  9 10 11 12   4  5  6  7  8  9 10   8  9 10 11 12 13 14  
-13 14 15 16 17 18 19  11 12 13 14 15 16 17  15 16 17 18 19 20 21  
-20 21 22 23 24 25 26  18 19 20 21 22 23 24  22 23 24 25 26 27 28  
-27 28 29 30           25 26 27 28 29 30 31  29 30m                
-                                                                  
-
-        July                 August              September        
-Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
-       1  2  3  4  5                  1  2      1  2  3  4  5  6  
- 6  7  8  9 10 11 12   3  4  5  6  7  8  9   7  8  9 10 11 12 13  
-13 14 15 16 17 18 19  10 11 12 13 14 15 16  14 15 16 17 18 19 20  
-20 21 22 23 24 25 26  17 18 19 20 21 22 23  21 22 23 24 25 26 27  
-27 28 29 30 31m       24 25 26 27 28 29 30  28 29 30m             
-                      31m                                         
-
-      October               November              December        
-Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
-          1  2  3  4                     1      1  2  3  4  5  6  
- 5  6  7  8  9 10 11   2  3  4  5  6  7  8   7  8  9 10 11 12 13  
-12 13 14 15 16 17 18   9 10 11 12 13 14 15  14 15 16 17d18d19d20  
-19 20 21 22 23 24 25  16 17 18 19 20 21 22  21d22d23d24d25d26d27d 
-26 27 28 29 30 31m    23 24 25 26 27 28 29  28d29d30d31d           
-                      30m                                          
+      January               February               March
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
+          1  2  3  4                     1                     1
+ 5  6  7  8  9 10 11   2  3  4  5  6  7  8   2  3  4  5  6  7  8
+12 13 14 15 16 17 18   9 10 11 12 13 14 15   9 10 11 12 13 14 15
+19 20 21 22 23 24 25  16 17 18 19 20 21 22  16 17 18 19 20 21 22
+26 27 28 29 30 31     23 24 25 26 27 28     23 24 25 26 27 28 29
+                                            30 31
+
+       April                  May                   June
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
+       1  2  3  4  5               1  2  3   1  2  3  4  5  6  7
+ 6  7  8  9 10 11 12   4  5  6  7  8  9 10   8  9 10 11 12 13 14
+13 14 15 16 17 18 19  11 12 13 14 15 16 17  15 16 17 18 19 20 21
+20 21 22 23 24 25 26  18 19 20 21 22 23 24  22 23 24 25 26 27 28
+27 28 29 30           25 26 27 28 29 30 31  29 30m
+
+
+        July                 August              September
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
+       1  2  3  4  5                  1  2      1  2  3  4  5  6
+ 6  7  8  9 10 11 12   3  4  5  6  7  8  9   7  8  9 10 11 12 13
+13 14 15 16 17 18 19  10 11 12 13 14 15 16  14 15 16 17 18 19 20
+20 21 22 23 24 25 26  17 18 19 20 21 22 23  21 22 23 24 25 26 27
+27 28 29 30 31m       24 25 26 27 28 29 30  28 29 30m
+                      31m
+
+      October               November              December
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
+          1  2  3  4                     1      1  2  3  4  5  6
+ 5  6  7  8  9 10 11   2  3  4  5  6  7  8   7  8  9 10 11 12 13
+12 13 14 15 16 17 18   9 10 11 12 13 14 15  14 15 16 17d18d19d20
+19 20 21 22 23 24 25  16 17 18 19 20 21 22  21d22d23d24d25d26d27d
+26 27 28 29 30 31m    23 24 25 26 27 28 29  28d29d30d31d
+                      30m
 
 List view
 ---------

src/borg/archiver.py

@@ -3,7 +3,6 @@ import collections
 import functools
 import hashlib
 import inspect
-import io
 import logging
 import os
 import re
@@ -41,6 +40,7 @@ from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
 from .helpers import log_multi
 from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
 from .helpers import signal_handler
+from .helpers import ErrorIgnoringTextIOWrapper
 from .item import Item
 from .key import key_creator, RepoKey, PassphraseKey
 from .platform import get_flags
@@ -73,13 +73,16 @@ def with_repository(fake=False, create=False, lock=True, exclusive=False, manife
         @functools.wraps(method)
         def wrapper(self, args, **kwargs):
             location = args.location  # note: 'location' must be always present in args
+            append_only = getattr(args, 'append_only', False)
             if argument(args, fake):
                 return method(self, args, repository=None, **kwargs)
             elif location.proto == 'ssh':
-                repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args)
+                repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock,
+                                              append_only=append_only, args=args)
             else:
                 repository = Repository(location.path, create=create, exclusive=argument(args, exclusive),
-                                        lock_wait=self.lock_wait, lock=lock)
+                                        lock_wait=self.lock_wait, lock=lock,
+                                        append_only=append_only)
             with repository:
                 if manifest or cache:
                     kwargs['manifest'], kwargs['key'] = Manifest.load(repository)
@@ -1047,7 +1050,7 @@ class Archiver:
         Cache.break_lock(repository)
         return self.exit_code
 
-    helptext = {}
+    helptext = collections.OrderedDict()
     helptext['patterns'] = textwrap.dedent('''
         Exclusion patterns support four separate styles, fnmatch, shell, regular
         expressions and path prefixes. By default, fnmatch is used. If followed
@@ -1304,6 +1307,8 @@ class Archiver:
         subparser.add_argument('-e', '--encryption', dest='encryption',
                                choices=('none', 'keyfile', 'repokey'), default='repokey',
                                help='select encryption key mode (default: "%(default)s")')
+        subparser.add_argument('-a', '--append-only', dest='append_only', action='store_true',
+                               help='create an append-only mode repository')
 
         check_epilog = textwrap.dedent("""
         The check command verifies the consistency of a repository and the corresponding archives.
@@ -1796,7 +1801,7 @@ class Archiver:
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='break repository and cache locks')
         subparser.set_defaults(func=self.do_break_lock)
-        subparser.add_argument('location', metavar='REPOSITORY',
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False),
                                help='repository for which to break the locks')
 
@@ -2251,8 +2256,8 @@ def main():  # pragma: no cover
 
     # Make sure stdout and stderr have errors='replace' to avoid unicode
     # issues when print()-ing unicode file names
-    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True)
-    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True)
+    sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True)
+    sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True)
     setup_signal_handlers()
     archiver = Archiver()
     msg = None

src/borg/helpers.py

@@ -2,6 +2,7 @@ import argparse
 import grp
 import hashlib
 import logging
+import io
 import os
 import os.path
 import platform
@@ -1546,3 +1547,27 @@ def signal_handler(signo, handler):
         yield
     finally:
         signal.signal(signo, old_signal_handler)
+
+
+class ErrorIgnoringTextIOWrapper(io.TextIOWrapper):
+    def read(self, n):
+        if not self.closed:
+            try:
+                return super().read(n)
+            except BrokenPipeError:
+                try:
+                    super().close()
+                except OSError:
+                    pass
+        return ''
+
+    def write(self, s):
+        if not self.closed:
+            try:
+                return super().write(s)
+            except BrokenPipeError:
+                try:
+                    super().close()
+                except OSError:
+                    pass
+        return len(s)

src/borg/remote.py

@@ -117,7 +117,7 @@ class RepositoryServer:  # pragma: no cover
     def negotiate(self, versions):
         return RPC_PROTOCOL_VERSION
 
-    def open(self, path, create=False, lock_wait=None, lock=True):
+    def open(self, path, create=False, lock_wait=None, lock=True, append_only=False):
         path = os.fsdecode(path)
         if path.startswith('/~'):
             path = os.path.join(get_home_dir(), path[2:])
@@ -128,7 +128,7 @@ class RepositoryServer:  # pragma: no cover
                     break
             else:
                 raise PathNotAllowed(path)
-        self.repository = Repository(path, create, lock_wait=lock_wait, lock=lock, append_only=self.append_only)
+        self.repository = Repository(path, create, lock_wait=lock_wait, lock=lock, append_only=self.append_only or append_only)
         self.repository.__enter__()  # clean exit handled by serve() method
         return self.repository.id
 
@@ -137,10 +137,14 @@ class RemoteRepository:
     extra_test_args = []
 
     class RPCError(Exception):
-        def __init__(self, name):
+        def __init__(self, name, remote_type):
             self.name = name
+            self.remote_type = remote_type
 
-    def __init__(self, location, create=False, lock_wait=None, lock=True, args=None):
+    class NoAppendOnlyOnServer(Error):
+        """Server does not support --append-only."""
+
+    def __init__(self, location, create=False, lock_wait=None, lock=True, append_only=False, args=None):
         self.location = self._location = location
         self.preload_ids = []
         self.msgid = 0
@@ -176,7 +180,17 @@ class RemoteRepository:
         if version != RPC_PROTOCOL_VERSION:
             raise Exception('Server insisted on using unsupported protocol version %d' % version)
         try:
-            self.id = self.call('open', self.location.path, create, lock_wait, lock)
+            # Because of protocol versions, only send append_only if necessary
+            if append_only:
+                try:
+                    self.id = self.call('open', self.location.path, create, lock_wait, lock, append_only)
+                except self.RPCError as err:
+                    if err.remote_type == 'TypeError':
+                        raise self.NoAppendOnlyOnServer() from err
+                    else:
+                        raise
+            else:
+                self.id = self.call('open', self.location.path, create, lock_wait, lock)
         except Exception:
             self.close()
             raise
@@ -272,7 +286,7 @@ class RemoteRepository:
             elif error == b'InvalidRPCMethod':
                 raise InvalidRPCMethod(*res)
             else:
-                raise self.RPCError(res.decode('utf-8'))
+                raise self.RPCError(res.decode('utf-8'), error.decode('utf-8'))
 
         calls = list(calls)
         waiting_for = []

src/borg/repository.py

@@ -154,7 +154,7 @@ class Repository:
         config.set('repository', 'version', '1')
         config.set('repository', 'segments_per_dir', str(DEFAULT_SEGMENTS_PER_DIR))
         config.set('repository', 'max_segment_size', str(DEFAULT_MAX_SEGMENT_SIZE))
-        config.set('repository', 'append_only', '0')
+        config.set('repository', 'append_only', str(int(self.append_only)))
         config.set('repository', 'id', bin_to_hex(os.urandom(32)))
         self.save_config(path, config)
 

tox.ini

@@ -15,4 +15,4 @@ passenv = *
 [testenv:flake8]
 changedir =
 deps = flake8
-commands = flake8 
+commands = flake8