Merge branch '1.0-maint'

Vagrantfile

@@ -109,7 +109,6 @@ def packages_openbsd
     pkg_add lz4
     # pkg_add fuse  # does not install, sdl dependency missing
     pkg_add git  # no fakeroot
-    pkg_add python-3.4.2
     pkg_add py3-setuptools
     ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
     ln -sf /usr/local/bin/python3.4 /usr/local/bin/python
@@ -166,7 +165,7 @@ def install_pythons(boxname)
     . ~/.bash_profile
     pyenv install 3.4.0  # tests
     pyenv install 3.5.0  # tests
-    pyenv install 3.5.1  # binary build, use latest 3.5.x release
+    pyenv install 3.5.2  # binary build, use latest 3.5.x release
     pyenv rehash
   EOF
 end
@@ -184,8 +183,8 @@ def build_pyenv_venv(boxname)
     . ~/.bash_profile
     cd /vagrant/borg
     # use the latest 3.5 release
-    pyenv global 3.5.1
-    pyenv virtualenv 3.5.1 borg-env
+    pyenv global 3.5.2
+    pyenv virtualenv 3.5.2 borg-env
     ln -s ~/.pyenv/versions/borg-env .
   EOF
 end
@@ -207,6 +206,22 @@ def install_borg(boxname)
   EOF
 end
 
+def install_borg_no_fuse(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    . borg-env/bin/activate
+    pip install -U wheel  # upgrade wheel, too old for 3.5
+    cd borg
+    # clean up (wrong/outdated) stuff we likely got via rsync:
+    rm -f borg/*.so borg/*.cpy*
+    rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
+    rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
+    pip install -r requirements.d/development.txt
+    pip install -e .
+  EOF
+end
+
 def install_pyinstaller(boxname)
   return <<-EOF
     . ~/.bash_profile
@@ -417,13 +432,13 @@ Vagrant.configure(2) do |config|
   end
 
   config.vm.define "openbsd64" do |b|
-    b.vm.box = "bodgit/openbsd-5.7-amd64"
+    b.vm.box = "kaorimatz/openbsd-5.9-amd64"
     b.vm.provider :virtualbox do |v|
       v.memory = 768
     end
     b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("openbsd64")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
   end
 

docs/misc/prune-example.txt

@@ -0,0 +1,93 @@
+borg prune visualized
+=====================
+
+Assume it is 2016-01-01, today's backup has not yet been made and you have
+created at least one backup on each day in 2015 except on 2015-12-20 (no
+backup made on that day).
+
+This is what borg prune --keep-daily 14 --keep-monthly 6 would keep.
+
+Backups kept by the --keep-daily rule are marked by a "d" to the right,
+backups kept by the --keep-monthly rule are marked by a "m" to the right.
+
+Calendar view
+-------------
+
+                            2015
+      January               February               March          
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
+          1  2  3  4                     1                     1  
+ 5  6  7  8  9 10 11   2  3  4  5  6  7  8   2  3  4  5  6  7  8  
+12 13 14 15 16 17 18   9 10 11 12 13 14 15   9 10 11 12 13 14 15  
+19 20 21 22 23 24 25  16 17 18 19 20 21 22  16 17 18 19 20 21 22  
+26 27 28 29 30 31     23 24 25 26 27 28     23 24 25 26 27 28 29  
+                                            30 31                 
+
+       April                  May                   June          
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
+       1  2  3  4  5               1  2  3   1  2  3  4  5  6  7  
+ 6  7  8  9 10 11 12   4  5  6  7  8  9 10   8  9 10 11 12 13 14  
+13 14 15 16 17 18 19  11 12 13 14 15 16 17  15 16 17 18 19 20 21  
+20 21 22 23 24 25 26  18 19 20 21 22 23 24  22 23 24 25 26 27 28  
+27 28 29 30           25 26 27 28 29 30 31  29 30m                
+                                                                  
+
+        July                 August              September        
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
+       1  2  3  4  5                  1  2      1  2  3  4  5  6  
+ 6  7  8  9 10 11 12   3  4  5  6  7  8  9   7  8  9 10 11 12 13  
+13 14 15 16 17 18 19  10 11 12 13 14 15 16  14 15 16 17 18 19 20  
+20 21 22 23 24 25 26  17 18 19 20 21 22 23  21 22 23 24 25 26 27  
+27 28 29 30 31m       24 25 26 27 28 29 30  28 29 30m             
+                      31m                                         
+
+      October               November              December        
+Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  
+          1  2  3  4                     1      1  2  3  4  5  6  
+ 5  6  7  8  9 10 11   2  3  4  5  6  7  8   7  8  9 10 11 12 13  
+12 13 14 15 16 17 18   9 10 11 12 13 14 15  14 15 16 17d18d19d20  
+19 20 21 22 23 24 25  16 17 18 19 20 21 22  21d22d23d24d25d26d27d 
+26 27 28 29 30 31m    23 24 25 26 27 28 29  28d29d30d31d           
+                      30m                                          
+
+List view
+---------
+
+--keep-daily 14     --keep-monthly 6
+-------------------------------------------------
+ 1. 2015-12-31          (2015-12-31 kept by daily rule)
+ 2. 2015-12-30       1. 2015-11-30
+ 3. 2015-12-29       2. 2015-10-31
+ 4. 2015-12-28       3. 2015-09-30
+ 5. 2015-12-27       4. 2015-08-31
+ 6. 2015-12-26       5. 2015-07-31
+ 7. 2015-12-25       6. 2015-06-30
+ 8. 2015-12-24
+ 9. 2015-12-23
+10. 2015-12-22
+11. 2015-12-21
+    (no backup made on 2015-12-20)
+12. 2015-12-19
+13. 2015-12-18
+14. 2015-12-17
+
+
+Notes
+-----
+
+2015-12-31 is kept due to the --keep-daily 14 rule (because it is applied
+first), not due to the --keep-monthly rule.
+
+Because of that, the --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and
+Jun. December is not considered for this rule, because that backup was already
+kept because of the daily rule.
+
+2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was
+made on 2015-12-20. If a backup had been made on that day, it would not keep
+the one from 2015-12-17.
+
+We did not include yearly, weekly, hourly, minutely or secondly rules to keep
+this example simple. They all work in basically the same way.
+
+The weekly rule is easy to understand roughly, but hard to understand in all
+details. If interested, read "ISO 8601:2000 standard week-based year".

docs/usage.rst

@@ -446,6 +446,8 @@ prefix "foo" if you do not also want to match "foobar".
 It is strongly recommended to always run ``prune --dry-run ...`` first so you
 will see what it would do without it actually doing anything.
 
+There is also a visualized prune example in ``docs/misc/prune-example.txt``.
+
 ::
 
     # Keep 7 end of day and 4 additional end of week archives.

src/borg/archive.py

@@ -4,6 +4,7 @@ import socket
 import stat
 import sys
 import time
+from contextlib import contextmanager
 from datetime import datetime, timezone
 from getpass import getuser
 from io import BytesIO
@@ -97,6 +98,37 @@ class Statistics:
             print(msg, file=stream or sys.stderr, end="\r", flush=True)
 
 
+class InputOSError(Exception):
+    """Wrapper for OSError raised while accessing input files."""
+    def __init__(self, os_error):
+        self.os_error = os_error
+        self.errno = os_error.errno
+        self.strerror = os_error.strerror
+        self.filename = os_error.filename
+
+    def __str__(self):
+        return str(self.os_error)
+
+
+@contextmanager
+def input_io():
+    """Context manager changing OSError to InputOSError."""
+    try:
+        yield
+    except OSError as os_error:
+        raise InputOSError(os_error) from os_error
+
+
+def input_io_iter(iterator):
+    while True:
+        try:
+            with input_io():
+                item = next(iterator)
+        except StopIteration:
+            return
+        yield item
+
+
 class DownloadPipeline:
 
     def __init__(self, repository, key):
@@ -560,13 +592,15 @@ Number of files: {0.stats.nfiles}'''.format(
         )
         if self.numeric_owner:
             attrs['user'] = attrs['group'] = None
-        xattrs = xattr.get_all(path, follow_symlinks=False)
+        with input_io():
+            xattrs = xattr.get_all(path, follow_symlinks=False)
         if xattrs:
             attrs['xattrs'] = StableDict(xattrs)
         bsdflags = get_flags(path, st)
         if bsdflags:
             attrs['bsdflags'] = bsdflags
-        acl_get(path, attrs, st, self.numeric_owner)
+        with input_io():
+            acl_get(path, attrs, st, self.numeric_owner)
         return attrs
 
     def process_dir(self, path, st):
@@ -601,7 +635,7 @@ Number of files: {0.stats.nfiles}'''.format(
         uid, gid = 0, 0
         fd = sys.stdin.buffer  # binary
         chunks = []
-        for data in self.chunker.chunkify(fd):
+        for data in input_io_iter(self.chunker.chunkify(fd)):
             chunks.append(cache.add_chunk(self.key.id_hash(data), Chunk(data), self.stats))
         self.stats.nfiles += 1
         t = int(time.time()) * 1000000000
@@ -654,10 +688,11 @@ Number of files: {0.stats.nfiles}'''.format(
         if chunks is None:
             compress = self.compression_decider1.decide(path)
             logger.debug('%s -> compression %s', path, compress['name'])
-            fh = Archive._open_rb(path)
+            with input_io():
+                fh = Archive._open_rb(path)
             with os.fdopen(fh, 'rb') as fd:
                 chunks = []
-                for data in self.chunker.chunkify(fd, fh):
+                for data in input_io_iter(self.chunker.chunkify(fd, fh)):
                     chunks.append(cache.add_chunk(self.key.id_hash(data),
                                                   Chunk(data, compress=compress),
                                                   self.stats))

src/borg/archiver.py

@@ -24,6 +24,7 @@ logger = create_logger()
 from . import __version__
 from . import helpers
 from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
+from .archive import InputOSError, CHUNKER_PARAMS
 from .cache import Cache
 from .constants import *  # NOQA
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
@@ -254,7 +255,7 @@ class Archiver:
                     if not dry_run:
                         try:
                             status = archive.process_stdin(path, cache)
-                        except OSError as e:
+                        except InputOSError as e:
                             status = 'E'
                             self.print_warning('%s: %s', path, e)
                     else:
@@ -312,7 +313,15 @@ class Archiver:
             return
         if st is None:
             try:
-                st = os.lstat(path)
+                # usually, do not follow symlinks (if we have a symlink, we want to
+                # backup it as such).
+                # but if we are in --read-special mode, we later process <path> as
+                # a regular file (we open and read the symlink target file's content).
+                # thus, in read_special mode, we also want to stat the symlink target
+                # file, for consistency. if we did not, we also have issues extracting
+                # this file, as it would be in the archive as a symlink, not as the
+                # target's file type (which could be e.g. a block device).
+                st = os.stat(path, follow_symlinks=read_special)
             except OSError as e:
                 self.print_warning('%s: %s', path, e)
                 return
@@ -330,7 +339,7 @@ class Archiver:
             if not dry_run:
                 try:
                     status = archive.process_file(path, st, cache, self.ignore_inode)
-                except OSError as e:
+                except InputOSError as e:
                     status = 'E'
                     self.print_warning('%s: %s', path, e)
         elif stat.S_ISDIR(st.st_mode):

src/borg/remote.py

@@ -249,6 +249,24 @@ class RemoteRepository:
                 del self.cache[args]
             return msgid
 
+        def handle_error(error, res):
+            if error == b'DoesNotExist':
+                raise Repository.DoesNotExist(self.location.orig)
+            elif error == b'AlreadyExists':
+                raise Repository.AlreadyExists(self.location.orig)
+            elif error == b'CheckNeeded':
+                raise Repository.CheckNeeded(self.location.orig)
+            elif error == b'IntegrityError':
+                raise IntegrityError(res)
+            elif error == b'PathNotAllowed':
+                raise PathNotAllowed(*res)
+            elif error == b'ObjectNotFound':
+                raise Repository.ObjectNotFound(res[0], self.location.orig)
+            elif error == b'InvalidRPCMethod':
+                raise InvalidRPCMethod(*res)
+            else:
+                raise self.RPCError(res.decode('utf-8'))
+
         calls = list(calls)
         waiting_for = []
         while wait or calls:
@@ -257,22 +275,7 @@ class RemoteRepository:
                     error, res = self.responses.pop(waiting_for[0])
                     waiting_for.pop(0)
                     if error:
-                        if error == b'DoesNotExist':
-                            raise Repository.DoesNotExist(self.location.orig)
-                        elif error == b'AlreadyExists':
-                            raise Repository.AlreadyExists(self.location.orig)
-                        elif error == b'CheckNeeded':
-                            raise Repository.CheckNeeded(self.location.orig)
-                        elif error == b'IntegrityError':
-                            raise IntegrityError(res)
-                        elif error == b'PathNotAllowed':
-                            raise PathNotAllowed(*res)
-                        elif error == b'ObjectNotFound':
-                            raise Repository.ObjectNotFound(res[0], self.location.orig)
-                        elif error == b'InvalidRPCMethod':
-                            raise InvalidRPCMethod(*res)
-                        else:
-                            raise self.RPCError(res.decode('utf-8'))
+                        handle_error(error, res)
                     else:
                         yield res
                         if not waiting_for and not calls:
@@ -298,6 +301,8 @@ class RemoteRepository:
                         type, msgid, error, res = unpacked
                         if msgid in self.ignore_responses:
                             self.ignore_responses.remove(msgid)
+                            if error:
+                                handle_error(error, res)
                         else:
                             self.responses[msgid] = error, res
                 elif fd is self.stderr_fd:

src/borg/testsuite/archive.py

@@ -7,6 +7,7 @@ import pytest
 import msgpack
 
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
+from ..archive import InputOSError, input_io, input_io_iter
 from ..item import Item
 from ..key import PlaintextKey
 from ..helpers import Manifest
@@ -216,3 +217,27 @@ def test_key_length_msgpacked_items():
     data = {key: b''}
     item_keys_serialized = [msgpack.packb(key), ]
     assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
+
+
+def test_input_io():
+    with pytest.raises(InputOSError):
+        with input_io():
+            raise OSError(123)
+
+
+def test_input_io_iter():
+    class Iterator:
+        def __init__(self, exc):
+            self.exc = exc
+
+        def __next__(self):
+            raise self.exc()
+
+    oserror_iterator = Iterator(OSError)
+    with pytest.raises(InputOSError):
+        for _ in input_io_iter(oserror_iterator):
+            pass
+
+    normal_iterator = Iterator(StopIteration)
+    for _ in input_io_iter(normal_iterator):
+        assert False, 'StopIteration handled incorrectly'