Merge pull request #2284 from enkore/cp1

picked cherries for 1.0-maint

borg/archive.py

@@ -19,11 +19,11 @@ from . import xattr
 from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \
     parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \
     Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
-    ProgressIndicatorPercent, IntegrityError
+    ProgressIndicatorPercent, IntegrityError, set_ec, EXIT_WARNING
 from .platform import acl_get, acl_set
 from .chunker import Chunker
 from .hashindex import ChunkIndex
-from .repository import Repository
+from .repository import Repository, LIST_SCAN_LIMIT
 
 import msgpack
 
@@ -501,11 +501,20 @@ Number of files: {0.stats.nfiles}'''.format(
             try:
                 xattr.setxattr(fd or path, k, v, follow_symlinks=False)
             except OSError as e:
-                if e.errno not in (errno.ENOTSUP, errno.EACCES):
-                    # only raise if the errno is not on our ignore list:
-                    # ENOTSUP == xattrs not supported here
-                    # EACCES == permission denied to set this specific xattr
-                    #           (this may happen related to security.* keys)
+                if e.errno == errno.E2BIG:
+                    # xattr is too big
+                    logger.warning('%s: Value or key of extended attribute %s is too big for this filesystem' %
+                                   (path, k.decode()))
+                    set_ec(EXIT_WARNING)
+                elif e.errno == errno.ENOTSUP:
+                    # xattrs not supported here
+                    logger.warning('%s: Extended attributes are not supported on this filesystem' % path)
+                    set_ec(EXIT_WARNING)
+                elif e.errno == errno.EACCES:
+                    # permission denied to set this specific xattr (this may happen related to security.* keys)
+                    logger.warning('%s: Permission denied when setting extended attribute %s' % (path, k.decode()))
+                    set_ec(EXIT_WARNING)
+                else:
                     raise
 
     def rename(self, name):
@@ -533,7 +542,7 @@ Number of files: {0.stats.nfiles}'''.format(
                 raise ChunksIndexError(cid)
             except Repository.ObjectNotFound as e:
                 # object not in repo - strange, but we wanted to delete it anyway.
-                if not forced:
+                if forced == 0:
                     raise
                 error = True
 
@@ -555,14 +564,14 @@ Number of files: {0.stats.nfiles}'''.format(
                 except (TypeError, ValueError):
                     # if items metadata spans multiple chunks and one chunk got dropped somehow,
                     # it could be that unpacker yields bad types
-                    if not forced:
+                    if forced == 0:
                         raise
                     error = True
             if progress:
                 pi.finish()
         except (msgpack.UnpackException, Repository.ObjectNotFound):
             # items metadata corrupted
-            if not forced:
+            if forced == 0:
                 raise
             error = True
         # in forced delete mode, we try hard to delete at least the manifest entry,
@@ -882,7 +891,7 @@ class ArchiveChecker:
         self.chunks = ChunkIndex(capacity)
         marker = None
         while True:
-            result = self.repository.list(limit=10000, marker=marker)
+            result = self.repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
             if not result:
                 break
             marker = result[-1]
@@ -984,6 +993,13 @@ class ArchiveChecker:
             Missing file chunks will be replaced with new chunks of the same length containing all zeros.
             If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one.
             """
+            def replacement_chunk(size):
+                data = bytes(size)
+                chunk_id = self.key.id_hash(data)
+                cdata = self.key.encrypt(data)
+                csize = len(cdata)
+                return chunk_id, size, csize, cdata
+
             offset = 0
             chunk_list = []
             chunks_replaced = False
@@ -1000,17 +1016,21 @@ class ArchiveChecker:
                                      'Replacing with all-zero chunk.'.format(
                                      item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size))
                         self.error_found = chunks_replaced = True
-                        data = bytes(size)
-                        chunk_id = self.key.id_hash(data)
-                        cdata = self.key.encrypt(data)
-                        csize = len(cdata)
+                        chunk_id, size, csize, cdata = replacement_chunk(size)
                         add_reference(chunk_id, size, csize, cdata)
                     else:
                         logger.info('{}: Previously missing file chunk is still missing (Byte {}-{}). '
                                     'It has a all-zero replacement chunk already.'.format(
                                     item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size))
                         chunk_id, size, csize = chunk_current
-                        add_reference(chunk_id, size, csize)
+                        if chunk_id in self.chunks:
+                            add_reference(chunk_id, size, csize)
+                        else:
+                            logger.warning('{}: Missing all-zero replacement chunk detected (Byte {}-{}). '
+                                           'Generating new replacement chunk.'.format(item.path, offset, offset + size))
+                            self.error_found = chunks_replaced = True
+                            chunk_id, size, csize, cdata = replacement_chunk(size)
+                            add_reference(chunk_id, size, csize, cdata)
                 else:
                     if chunk_current == chunk_healthy:
                         # normal case, all fine.

borg/archiver.py

@@ -23,13 +23,13 @@ from .helpers import Error, location_validator, archivename_validator, format_li
     PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, get_cache_dir, prune_within, prune_split, \
     Manifest, NoManifestError, remove_surrogates, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
-    EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper
+    EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper, set_ec
 from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm
 from .logger import create_logger, setup_logging
 logger = create_logger()
 from .compress import Compressor
 from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
-from .repository import Repository
+from .repository import Repository, LIST_SCAN_LIMIT
 from .cache import Cache
 from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
 from .keymanager import KeyManager
@@ -331,92 +331,90 @@ class Archiver:
     def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present,
                  keep_tag_files, skip_inodes, path, restrict_dev,
                  read_special=False, dry_run=False):
+        """
+        Process *path* recursively according to the various parameters.
+
+        This should only raise on critical errors. Per-item errors must be handled within this method.
+        """
         if not matcher.match(path):
             return
 
         try:
-            st = os.lstat(path)
-        except OSError as e:
-            self.print_warning('%s: %s', path, e)
-            return
-        if (st.st_ino, st.st_dev) in skip_inodes:
-            return
-        # Entering a new filesystem?
-        if restrict_dev is not None and st.st_dev != restrict_dev:
-            return
-        status = None
-        # Ignore if nodump flag is set
-        if has_lchflags and (st.st_flags & stat.UF_NODUMP):
-            return
-        if stat.S_ISREG(st.st_mode):
-            if not dry_run:
-                try:
-                    status = archive.process_file(path, st, cache, self.ignore_inode)
-                except BackupOSError as e:
-                    status = 'E'
-                    self.print_warning('%s: %s', path, e)
-        elif stat.S_ISDIR(st.st_mode):
-            tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present)
-            if tag_paths:
-                if keep_tag_files and not dry_run:
-                    archive.process_dir(path, st)
-                    for tag_path in tag_paths:
-                        self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
-                                      keep_tag_files, skip_inodes, tag_path, restrict_dev,
-                                      read_special=read_special, dry_run=dry_run)
+            with backup_io():
+                st = os.lstat(path)
+            if (st.st_ino, st.st_dev) in skip_inodes:
                 return
-            if not dry_run:
-                status = archive.process_dir(path, st)
-            try:
-                entries = os.listdir(path)
-            except OSError as e:
-                status = 'E'
-                self.print_warning('%s: %s', path, e)
-            else:
+            # Entering a new filesystem?
+            if restrict_dev is not None and st.st_dev != restrict_dev:
+                return
+            status = None
+            # Ignore if nodump flag is set
+            if has_lchflags and (st.st_flags & stat.UF_NODUMP):
+                return
+            if stat.S_ISREG(st.st_mode):
+                if not dry_run:
+                    status = archive.process_file(path, st, cache, self.ignore_inode)
+            elif stat.S_ISDIR(st.st_mode):
+                tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present)
+                if tag_paths:
+                    if keep_tag_files and not dry_run:
+                        archive.process_dir(path, st)
+                        for tag_path in tag_paths:
+                            self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
+                                          keep_tag_files, skip_inodes, tag_path, restrict_dev,
+                                          read_special=read_special, dry_run=dry_run)
+                    return
+                if not dry_run:
+                    status = archive.process_dir(path, st)
+                with backup_io():
+                    entries = os.listdir(path)
                 for filename in sorted(entries):
                     entry_path = os.path.normpath(os.path.join(path, filename))
                     self._process(archive, cache, matcher, exclude_caches, exclude_if_present,
                                   keep_tag_files, skip_inodes, entry_path, restrict_dev,
                                   read_special=read_special, dry_run=dry_run)
-        elif stat.S_ISLNK(st.st_mode):
-            if not dry_run:
-                if not read_special:
-                    status = archive.process_symlink(path, st)
-                else:
-                    try:
-                        st_target = os.stat(path)
-                    except OSError:
-                        special = False
+            elif stat.S_ISLNK(st.st_mode):
+                if not dry_run:
+                    if not read_special:
+                        status = archive.process_symlink(path, st)
                     else:
-                        special = is_special(st_target.st_mode)
-                    if special:
-                        status = archive.process_file(path, st_target, cache)
+                        try:
+                            st_target = os.stat(path)
+                        except OSError:
+                            special = False
+                        else:
+                            special = is_special(st_target.st_mode)
+                        if special:
+                            status = archive.process_file(path, st_target, cache)
+                        else:
+                            status = archive.process_symlink(path, st)
+            elif stat.S_ISFIFO(st.st_mode):
+                if not dry_run:
+                    if not read_special:
+                        status = archive.process_fifo(path, st)
                     else:
-                        status = archive.process_symlink(path, st)
-        elif stat.S_ISFIFO(st.st_mode):
-            if not dry_run:
-                if not read_special:
-                    status = archive.process_fifo(path, st)
-                else:
-                    status = archive.process_file(path, st, cache)
-        elif stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
-            if not dry_run:
-                if not read_special:
-                    status = archive.process_dev(path, st)
-                else:
-                    status = archive.process_file(path, st, cache)
-        elif stat.S_ISSOCK(st.st_mode):
-            # Ignore unix sockets
-            return
-        elif stat.S_ISDOOR(st.st_mode):
-            # Ignore Solaris doors
-            return
-        elif stat.S_ISPORT(st.st_mode):
-            # Ignore Solaris event ports
-            return
-        else:
-            self.print_warning('Unknown file type: %s', path)
-            return
+                        status = archive.process_file(path, st, cache)
+            elif stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
+                if not dry_run:
+                    if not read_special:
+                        status = archive.process_dev(path, st)
+                    else:
+                        status = archive.process_file(path, st, cache)
+            elif stat.S_ISSOCK(st.st_mode):
+                # Ignore unix sockets
+                return
+            elif stat.S_ISDOOR(st.st_mode):
+                # Ignore Solaris doors
+                return
+            elif stat.S_ISPORT(st.st_mode):
+                # Ignore Solaris event ports
+                return
+            else:
+                self.print_warning('Unknown file type: %s', path)
+                return
+        except BackupOSError as e:
+            self.print_warning('%s: %s', path, e)
+            status = 'E'
         # Status output
         if status is None:
             if not dry_run:
@@ -505,9 +503,23 @@ class Archiver:
     def do_delete(self, args, repository):
         """Delete an existing repository or archive"""
         if args.location.archive:
+            archive_name = args.location.archive
             manifest, key = Manifest.load(repository)
+
+            if args.forced == 2:
+                try:
+                    del manifest.archives[archive_name]
+                except KeyError:
+                    raise Archive.DoesNotExist(archive_name)
+                logger.info('Archive deleted.')
+                manifest.write()
+                # note: might crash in compact() after committing the repo
+                repository.commit()
+                logger.info('Done. Run "borg check --repair" to clean up the mess.')
+                return self.exit_code
+
             with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
-                archive = Archive(repository, key, manifest, args.location.archive, cache=cache)
+                archive = Archive(repository, key, manifest, archive_name, cache=cache)
                 stats = Statistics()
                 archive.delete(stats, progress=args.progress, forced=args.forced)
                 manifest.write()
@@ -815,7 +827,7 @@ class Archiver:
         marker = None
         i = 0
         while True:
-            result = repository.list(limit=10000, marker=marker)
+            result = repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
             if not result:
                 break
             marker = result[-1]
@@ -1556,8 +1568,9 @@ class Archiver:
                                action='store_true', default=False,
                                help='delete only the local cache for the given repository')
         subparser.add_argument('--force', dest='forced',
-                               action='store_true', default=False,
-                               help='force deletion of corrupted archives')
+                               action='count', default=0,
+                               help='force deletion of corrupted archives, '
+                                    'use --force --force in case --force does not work.')
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
                                help='work slower, but using less space')
@@ -2077,7 +2090,7 @@ class Archiver:
         check_extension_modules()
         if is_slow_msgpack():
             logger.warning("Using a pure-python msgpack! This will result in lower performance.")
-        return func(args)
+        return set_ec(func(args))
 
 
 def sig_info_handler(sig_no, stack):  # pragma: no cover

borg/compress.pyx

@@ -4,7 +4,9 @@ try:
 except ImportError:
     lzma = None
 
-from .helpers import Buffer
+from .helpers import Buffer, DecompressionError
+
+API_VERSION = '1.0_01'
 
 cdef extern from "lz4.h":
     int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
@@ -110,7 +112,7 @@ class LZ4(CompressorBase):
                 break
             if osize > 2 ** 30:
                 # this is insane, get out of here
-                raise Exception('lz4 decompress failed')
+                raise DecompressionError('lz4 decompress failed')
             # likely the buffer was too small, get a bigger one:
             osize = int(1.5 * osize)
         return dest[:rsize]
@@ -136,7 +138,10 @@ class LZMA(CompressorBase):
 
     def decompress(self, data):
         data = super().decompress(data)
-        return lzma.decompress(data)
+        try:
+            return lzma.decompress(data)
+        except lzma.LZMAError as e:
+            raise DecompressionError(str(e)) from None
 
 
 class ZLIB(CompressorBase):
@@ -165,7 +170,10 @@ class ZLIB(CompressorBase):
 
     def decompress(self, data):
         # note: for compatibility no super call, do not strip ID bytes
-        return zlib.decompress(data)
+        try:
+            return zlib.decompress(data)
+        except zlib.error as e:
+            raise DecompressionError(str(e)) from None
 
 
 COMPRESSOR_TABLE = {

borg/hashindex.pyx

@@ -216,7 +216,7 @@ cdef class ChunkIndex(IndexBase):
         if not data:
             raise KeyError(key)
         cdef uint32_t refcount = _le32toh(data[0])
-        assert refcount <= _MAX_VALUE
+        assert refcount <= _MAX_VALUE, "invalid reference count"
         return refcount, _le32toh(data[1]), _le32toh(data[2])
 
     def __setitem__(self, key, value):
@@ -234,7 +234,7 @@ cdef class ChunkIndex(IndexBase):
         assert len(key) == self.key_size
         data = <uint32_t *>hashindex_get(self.index, <char *>key)
         if data != NULL:
-            assert data[0] <= _MAX_VALUE
+            assert _le32toh(data[0]) <= _MAX_VALUE, "invalid reference count"
         return data != NULL
 
     def incref(self, key):
@@ -312,8 +312,8 @@ cdef class ChunkIndex(IndexBase):
         if values:
             refcount1 = _le32toh(values[0])
             refcount2 = _le32toh(data[0])
-            assert refcount1 <= _MAX_VALUE
-            assert refcount2 <= _MAX_VALUE
+            assert refcount1 <= _MAX_VALUE, "invalid reference count"
+            assert refcount2 <= _MAX_VALUE, "invalid reference count"
             result64 = refcount1 + refcount2
             values[0] = _htole32(min(result64, _MAX_VALUE))
         else:

borg/helpers.py

@@ -45,6 +45,26 @@ EXIT_WARNING = 1  # reached normal end of operation, but there were issues
 EXIT_ERROR = 2  # terminated abruptly, did not reach end of operation
 
 
+'''
+The global exit_code variable is used so that modules other than archiver can increase the program exit code if a
+warning or error occured during their operation. This is different from archiver.exit_code, which is only accessible
+from the archiver object.
+'''
+exit_code = EXIT_SUCCESS
+
+
+def set_ec(ec):
+    '''
+    Sets the exit code of the program, if an exit code higher or equal than this is set, this does nothing. This
+    makes EXIT_ERROR override EXIT_WARNING, etc..
+
+    ec: exit code to set
+    '''
+    global exit_code
+    exit_code = max(exit_code, ec)
+    return exit_code
+
+
 class Error(Exception):
     """Error base class"""
 
@@ -74,6 +94,10 @@ class IntegrityError(ErrorWithTraceback):
     """Data integrity error: {}"""
 
 
+class DecompressionError(IntegrityError):
+    """Decompression error: {}"""
+
+
 class ExtensionModuleError(Error):
     """The Borg binary extension modules do not seem to be properly installed"""
 
@@ -87,11 +111,13 @@ class PlaceholderError(Error):
 
 
 def check_extension_modules():
-    from . import platform
+    from . import platform, compress
     if hashindex.API_VERSION != '1.0_01':
         raise ExtensionModuleError
     if chunker.API_VERSION != '1.0_01':
         raise ExtensionModuleError
+    if compress.API_VERSION != '1.0_01':
+        raise ExtensionModuleError
     if crypto.API_VERSION != '1.0_01':
         raise ExtensionModuleError
     if platform.API_VERSION != '1.0_01':
@@ -899,10 +925,17 @@ class Location:
 
     # path must not contain :: (it ends at :: or string end), but may contain single colons.
     # to avoid ambiguities with other regexes, it must also not start with ":" nor with "//" nor with "ssh://".
-    path_re = r"""
+    scp_path_re = r"""
         (?!(:|//|ssh://))                                   # not starting with ":" or // or ssh://
         (?P<path>([^:]|(:(?!:)))+)                          # any chars, but no "::"
         """
+
+    # file_path must not contain :: (it ends at :: or string end), but may contain single colons.
+    # it must start with a / and that slash is part of the path.
+    file_path_re = r"""
+        (?P<path>(([^/]*)/([^:]|(:(?!:)))+))                # start opt. servername, then /, then any chars, but no "::"
+        """
+
     # abs_path must not contain :: (it ends at :: or string end), but may contain single colons.
     # it must start with a / and that slash is part of the path.
     abs_path_re = r"""
@@ -927,7 +960,7 @@ class Location:
 
     file_re = re.compile(r"""
         (?P<proto>file)://                                  # file://
-        """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive
+        """ + file_path_re + optional_archive_re, re.VERBOSE)  # servername/path, path or path::archive
 
     # note: scp_re is also use for local pathes
     scp_re = re.compile(r"""
@@ -935,7 +968,7 @@ class Location:
             """ + optional_user_re + r"""                   # user@  (optional)
             (?P<host>[^:/]+):                               # host: (don't match / in host to disambiguate from file:)
         )?                                                  # user@host: part is optional
-        """ + path_re + optional_archive_re, re.VERBOSE)    # path with optional archive
+        """ + scp_path_re + optional_archive_re, re.VERBOSE)  # path with optional archive
 
     # get the repo from BORG_REPO env and the optional archive from param.
     # if the syntax requires giving REPOSITORY (see "borg mount"),

borg/remote.py

@@ -15,7 +15,7 @@ from . import __version__
 from .helpers import Error, IntegrityError, sysinfo
 from .helpers import replace_placeholders
 from .helpers import bin_to_hex
-from .repository import Repository
+from .repository import Repository, LIST_SCAN_LIMIT, MAX_OBJECT_SIZE
 from .logger import create_logger
 
 import msgpack
@@ -46,6 +46,27 @@ def os_write(fd, data):
     return amount
 
 
+def get_limited_unpacker(kind):
+    """return a limited Unpacker because we should not trust msgpack data received from remote"""
+    args = dict(use_list=False,  # return tuples, not lists
+                max_bin_len=0,  # not used
+                max_ext_len=0,  # not used
+                max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
+                max_str_len=MAX_OBJECT_SIZE,  # a chunk or other repo object
+                )
+    if kind == 'server':
+        args.update(dict(max_array_len=100,  # misc. cmd tuples
+                         max_map_len=100,  # misc. cmd dicts
+                         ))
+    elif kind == 'client':
+        args.update(dict(max_array_len=LIST_SCAN_LIMIT,  # result list from repo.list() / .scan()
+                         max_map_len=100,  # misc. result dicts
+                         ))
+    else:
+        raise ValueError('kind must be "server" or "client"')
+    return msgpack.Unpacker(**args)
+
+
 class ConnectionClosed(Error):
     """Connection closed by remote host"""
 
@@ -115,7 +136,7 @@ class RepositoryServer:  # pragma: no cover
         # Make stderr blocking
         fl = fcntl.fcntl(stderr_fd, fcntl.F_GETFL)
         fcntl.fcntl(stderr_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK)
-        unpacker = msgpack.Unpacker(use_list=False)
+        unpacker = get_limited_unpacker('server')
         while True:
             r, w, es = select.select([stdin_fd], [], [], 10)
             if r:
@@ -205,7 +226,7 @@ class RemoteRepository:
         self.cache = {}
         self.ignore_responses = set()
         self.responses = {}
-        self.unpacker = msgpack.Unpacker(use_list=False)
+        self.unpacker = get_limited_unpacker('client')
         self.p = None
         testing = location.host == '__testsuite__'
         borg_cmd = self.borg_cmd(args, testing)

borg/repository.py

@@ -26,6 +26,8 @@ TAG_PUT = 0
 TAG_DELETE = 1
 TAG_COMMIT = 2
 
+LIST_SCAN_LIMIT = 10000  # repo.list() / .scan() result count limit the borg client uses
+
 
 class Repository:
     """Filesystem based transactional key value store

borg/testsuite/archiver.py

@@ -19,7 +19,7 @@ from hashlib import sha256
 import msgpack
 import pytest
 
-from .. import xattr
+from .. import xattr, helpers
 from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP, flags_noatime, flags_normal
 from ..archiver import Archiver
 from ..cache import Cache
@@ -68,6 +68,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
             if archiver is None:
                 archiver = Archiver()
             archiver.exit_code = EXIT_SUCCESS
+            helpers.exit_code = EXIT_SUCCESS
             args = archiver.parse_args(list(args))
             ret = archiver.run(args)
             return ret, output.getvalue()
@@ -245,7 +246,7 @@ class ArchiverTestCaseBase(BaseTestCase):
         return output
 
     def create_src_archive(self, name):
-        self.cmd('create', self.repository_location + '::' + name, src_dir)
+        self.cmd('create', '--compression=lz4', self.repository_location + '::' + name, src_dir)
 
     def open_archive(self, name):
         repository = Repository(self.repository_path, exclusive=True)
@@ -780,6 +781,38 @@ class ArchiverTestCase(ArchiverTestCaseBase):
                 self.cmd('extract', self.repository_location + '::test')
             assert xattr.getxattr('input/file', 'security.capability') == capabilities
 
+    @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='xattr not supported on this system or on this version of'
+                                                         'fakeroot')
+    def test_extract_xattrs_errors(self):
+        def patched_setxattr_E2BIG(*args, **kwargs):
+            raise OSError(errno.E2BIG, 'E2BIG')
+
+        def patched_setxattr_ENOTSUP(*args, **kwargs):
+            raise OSError(errno.ENOTSUP, 'ENOTSUP')
+
+        def patched_setxattr_EACCES(*args, **kwargs):
+            raise OSError(errno.EACCES, 'EACCES')
+
+        self.create_regular_file('file')
+        xattr.setxattr('input/file', 'attribute', 'value')
+        self.cmd('init', self.repository_location, '-e' 'none')
+        self.cmd('create', self.repository_location + '::test', 'input')
+        with changedir('output'):
+            input_abspath = os.path.abspath('input/file')
+            with patch.object(xattr, 'setxattr', patched_setxattr_E2BIG):
+                out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING)
+                assert out == (input_abspath + ': Value or key of extended attribute attribute is too big for this '
+                                               'filesystem\n')
+            os.remove(input_abspath)
+            with patch.object(xattr, 'setxattr', patched_setxattr_ENOTSUP):
+                out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING)
+                assert out == (input_abspath + ': Extended attributes are not supported on this filesystem\n')
+            os.remove(input_abspath)
+            with patch.object(xattr, 'setxattr', patched_setxattr_EACCES):
+                out = self.cmd('extract', self.repository_location + '::test', exit_code=EXIT_WARNING)
+                assert out == (input_abspath + ': Permission denied when setting extended attribute attribute\n')
+            assert os.path.isfile(input_abspath)
+
     def test_path_normalization(self):
         self.cmd('init', self.repository_location)
         self.create_regular_file('dir1/dir2/file', size=1024 * 80)
@@ -881,6 +914,20 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         # Make sure the repo is gone
         self.assertFalse(os.path.exists(self.repository_path))
 
+    def test_delete_double_force(self):
+        self.cmd('init', '--encryption=none', self.repository_location)
+        self.create_src_archive('test')
+        with Repository(self.repository_path, exclusive=True) as repository:
+            manifest, key = Manifest.load(repository)
+            archive = Archive(repository, key, manifest, 'test')
+            id = archive.metadata[b'items'][0]
+            repository.put(id, b'corrupted items metadata stream chunk')
+            repository.commit()
+        self.cmd('delete', '--force', '--force', self.repository_location + '::test')
+        self.cmd('check', '--repair', self.repository_location)
+        output = self.cmd('list', self.repository_location)
+        self.assert_not_in('test', output)
+
     def test_corrupted_repository(self):
         self.cmd('init', self.repository_location)
         self.create_src_archive('test')

borg/testsuite/helpers.py

@@ -57,6 +57,11 @@ class TestLocationWithoutEnv:
         assert repr(Location('user@host:/some/path')) == \
             "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
 
+    def test_smb(self, monkeypatch):
+        monkeypatch.delenv('BORG_REPO', raising=False)
+        assert repr(Location('file:////server/share/path::archive')) == \
+            "Location(proto='file', user=None, host=None, port=None, path='//server/share/path', archive='archive')"
+
     def test_folder(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('path::archive')) == \

docs/development.rst

@@ -19,18 +19,7 @@ Some guidance for contributors:
 
 - discuss about changes on github issue tracker, IRC or mailing list
 
-- choose the branch you base your changesets on wisely:
-
-  - choose x.y-maint for stuff that should go into next x.y.z release
-    (it usually gets merged into master branch later also), like:
-
-    - bug fixes (code or docs)
-    - missing *important* (and preferably small) features
-    - docs rearrangements (so stuff stays in-sync to avoid merge
-      troubles in future)
-  - choose master if that does not apply, like for:
-
-    - developing new features
+- make your PRs on the ``master`` branch (see `Branching Model`_ for details)
 
 - do clean changesets:
 
@@ -56,6 +45,67 @@ Some guidance for contributors:
 
 - wait for review by other developers
 
+Branching model
+---------------
+
+Borg development happens on the ``master`` branch and uses GitHub pull
+requests (if you don't have GitHub or don't want to use it you can
+send smaller patches via the borgbackup :ref:`mailing_list` to the maintainers).
+
+Stable releases are maintained on maintenance branches named x.y-maint, eg.
+the maintenance branch of the 1.0.x series is 1.0-maint.
+
+Most PRs should be made against the ``master`` branch. Only if an
+issue affects **only** a particular maintenance branch a PR should be
+made against it directly.
+
+While discussing / reviewing a PR it will be decided whether the
+change should be applied to maintenance branch(es). Each maintenance
+branch has a corresponding *backport/x.y-maint* label, which will then
+be applied.
+
+Changes that are typically considered for backporting:
+
+- Data loss, corruption and inaccessibility fixes
+- Security fixes
+- Forward-compatibility improvements
+- Documentation corrections
+
+.. rubric:: Maintainer part
+
+From time to time a maintainer will backport the changes for a
+maintenance branch, typically before a release or if enough changes
+were collected:
+
+1. Notify others that you're doing this to avoid duplicate work.
+2. Branch a backporting branch off the maintenance branch.
+3. Cherry pick and backport the changes from each labelled PR, remove
+   the label for each PR you've backported.
+4. Make a PR of the backporting branch against the maintenance branch
+   for backport review. Mention the backported PRs in this PR, eg:
+
+       Includes changes from #2055 #2057 #2381
+
+   This way GitHub will automatically show in these PRs where they
+   were backported.
+
+.. rubric:: Historic model
+
+Previously (until release 1.0.10) Borg used a `"merge upwards"
+<https://git-scm.com/docs/gitworkflows#_merging_upwards>`_ model where
+most minor changes and fixes where committed to a maintenance branch
+(eg. 1.0-maint), and the maintenance branch(es) were regularly merged
+back into the main development branch. This became more and more
+troublesome due to merges growing more conflict-heavy and error-prone.
+
+Code and issues
+---------------
+
+Code is stored on Github, in the `Borgbackup organization
+<https://github.com/borgbackup/borg/>`_. `Issues
+<https://github.com/borgbackup/borg/issues>`_ and `pull requests
+<https://github.com/borgbackup/borg/pulls>`_ should be sent there as
+well. See also the :ref:`support` section for more details.
 
 Style guide
 -----------

docs/installation.rst

@@ -34,23 +34,35 @@ yet.
 Distribution Source                                        Command
 ============ ============================================= =======
 Arch Linux   `[community]`_                                ``pacman -S borg``
-Debian       `stretch`_, `unstable/sid`_                   ``apt install borgbackup``
+Debian       `Debian packages`_                            ``apt install borgbackup``
+Gentoo       `ebuild`_                                     ``emerge borgbackup``
+GNU Guix     `GNU Guix`_                                   ``guix package --install borg``
+Fedora/RHEL  `Fedora official repository`_                 ``dnf install borgbackup``
+FreeBSD      `FreeBSD ports`_                              ``cd /usr/ports/archivers/py-borgbackup && make install clean``
+Mageia       `cauldron`_                                   ``urpmi borgbackup``
 NetBSD       `pkgsrc`_                                     ``pkg_add py-borgbackup``
 NixOS        `.nix file`_                                  N/A
 OS X         `Brew cask`_                                  ``brew cask install borgbackup``
-Ubuntu       `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup``
-Ubuntu       `Trusty 14.04 (backport PPA)`_                ``apt install borgbackup``
+Raspbian     `Raspbian testing`_                           ``apt install borgbackup``
+Ubuntu       `Ubuntu packages`_, `Ubuntu PPA`_             ``apt install borgbackup``
 ============ ============================================= =======
 
 .. _[community]: https://www.archlinux.org/packages/?name=borg
-.. _stretch: https://packages.debian.org/stretch/borgbackup
-.. _unstable/sid: https://packages.debian.org/sid/borgbackup
+.. _Debian packages: https://packages.debian.org/search?keywords=borgbackup&searchon=names&exact=1&suite=all&section=all
+.. _Fedora official repository: https://apps.fedoraproject.org/packages/borgbackup
+.. _FreeBSD ports: http://www.freshports.org/archivers/py-borgbackup/
+.. _ebuild: https://packages.gentoo.org/packages/app-backup/borgbackup
+.. _GNU Guix: https://www.gnu.org/software/guix/package-list.html#borg
 .. _pkgsrc: http://pkgsrc.se/sysutils/py-borgbackup
+.. _cauldron: http://madb.mageia.org/package/show/application/0/release/cauldron/name/borgbackup
 .. _Xenial 16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup
 .. _Wily 15.10 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
 .. _Trusty 14.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
 .. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borg/default.nix
 .. _Brew cask: http://caskroom.io/
+.. _Raspbian testing: http://archive.raspbian.org/raspbian/pool/main/b/borgbackup/
+.. _Ubuntu packages: http://packages.ubuntu.com/xenial/borgbackup
+.. _Ubuntu PPA: https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup
 
 Please ask package maintainers to build a package or, if you can package /
 submit it yourself, please help us with that! See :issue:`105` on

docs/support.rst

@@ -28,6 +28,7 @@ nickname you get by typing "/nick mydesirednickname"):
 
 http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8
 
+.. _mailing_list:
 
 Mailing list
 ------------