Merge remote-tracking branch 'upstream/master'

.travis/install.sh

@@ -4,8 +4,6 @@ set -e
 set -x
 
 if [[ "$(uname -s)" == 'Darwin' ]]; then
-    brew update || brew update
-
     if [[ "${OPENSSL}" != "0.9.8" ]]; then
         brew outdated openssl || brew upgrade openssl
     fi

README.rst

@@ -89,9 +89,12 @@ Main features
 Easy to use
 ~~~~~~~~~~~
 
-Initialize a new backup repository and create a backup archive::
+Initialize a new backup repository (see ``borg init --help`` for encryption options)::
+
+    $ borg init -e repokey /path/to/repo
+
+Create a backup archive::
 
-    $ borg init /path/to/repo
     $ borg create /path/to/repo::Saturday1 ~/Documents
 
 Now doing another backup, just to show off the great deduplication::

docs/faq.rst

@@ -682,6 +682,31 @@ the corruption is caused by a one time event such as a power outage,
 running `borg check --repair` will fix most problems.
 
 
+Why isn't there more progress / ETA information displayed?
+----------------------------------------------------------
+
+Some borg runs take quite a bit, so it would be nice to see a progress display,
+maybe even including a ETA (expected time of "arrival" [here rather "completion"]).
+
+For some functionality, this can be done: if the total amount of work is more or
+less known, we can display progress. So check if there is a ``--progress`` option.
+
+But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do
+a single pass over the filesystem, so we do not know the total file count or data
+volume before reaching the end). Adding another pass just to determine that would
+take additional time and could be incorrect, if the filesystem is changing.
+
+Even if the fs does not change and we knew count and size of all files, we still
+could not compute the ``borg create`` ETA as we do not know the amount of changed
+chunks, how the bandwidth of source and destination or system performance might
+fluctuate.
+
+You see, trying to display ETA would be futile. The borg developers prefer to
+rather not implement progress / ETA display than doing futile attempts.
+
+See also: https://xkcd.com/612/
+
+
 Miscellaneous
 #############
 

docs/man_intro.rst

@@ -32,8 +32,8 @@ fully trusted targets.
 
 Borg stores a set of files in an *archive*. A *repository* is a collection
 of *archives*. The format of repositories is Borg-specific. Borg does not
-distinguish archives from each other in a any way other than their name,
-it does not matter when or where archives where created (eg. different hosts).
+distinguish archives from each other in any way other than their name,
+it does not matter when or where archives were created (e.g. different hosts).
 
 EXAMPLES
 --------
@@ -61,7 +61,7 @@ SEE ALSO
 
 `borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)`
 
-* Main web site https://borgbackup.readthedocs.org/
+* Main web site https://www.borgbackup.org/
 * Releases https://github.com/borgbackup/borg/releases
 * Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst
 * GitHub https://github.com/borgbackup/borg

docs/usage/general.rst

@@ -30,3 +30,11 @@ Common options
 All Borg commands share these options:
 
 .. include:: common-options.rst.inc
+
+Examples
+~~~~~~~~
+::
+
+    # Create an archive and log: borg version, files list, return code
+    $ borg create --show-version --list --show-rc /path/to/repo::my-files files
+

docs/usage/list.rst

@@ -19,7 +19,7 @@ Examples
     -rwxr-xr-x root   root       2140 Fri, 2015-03-27 20:24:22 bin/bzdiff
     ...
 
-    $ borg list /path/to/repo::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
+    $ borg list /path/to/repo::archiveA --format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}"
     drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 .
     drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 code
     drwxrwxr-x user   user          0 Sun, 2015-02-01 11:00:00 code/myproject

setup.py

@@ -12,7 +12,7 @@ from distutils.core import Command
 
 import textwrap
 
-min_python = (3, 4)
+min_python = (3, 5)
 my_python = sys.version_info
 
 if my_python < min_python:
@@ -40,6 +40,8 @@ extras_require = {
     # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
+    # llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0
+    # llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 2.0 will break API
     'fuse': ['llfuse<2.0', ],
 }
@@ -655,6 +657,13 @@ class build_man(Command):
     def gen_man_page(self, name, rst):
         from docutils.writers import manpage
         from docutils.core import publish_string
+        from docutils.nodes import inline
+        from docutils.parsers.rst import roles
+
+        def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
+            return [inline(rawtext, '#' + text)], []
+
+        roles.register_local_role('issue', issue)
         # We give the source_path so that docutils can find relative includes
         # as-if the document where located in the docs/ directory.
         man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer())

src/borg/archive.py

@@ -557,7 +557,7 @@ Utilization of max. archive size: {csize_max:.0%}
 
         original_path = original_path or item.path
         dest = self.cwd
-        if item.path.startswith(('/', '..')):
+        if item.path.startswith(('/', '../')):
             raise Exception('Path should be relative and local')
         path = os.path.join(dest, item.path)
         # Attempt to remove existing files, ignore errors on failure
@@ -965,7 +965,9 @@ class ChunksProcessor:
         length = len(item.chunks)
         # the item should only have the *additional* chunks we processed after the last partial item:
         item.chunks = item.chunks[from_chunk:]
-        item.get_size(memorize=True)
+        # for borg recreate, we already have a size member in the source item (giving the total file size),
+        # but we consider only a part of the file here, thus we must recompute the size from the chunks:
+        item.get_size(memorize=True, from_chunks=True)
         item.path += '.borg_part_%d' % number
         item.part = number
         number += 1
@@ -1813,7 +1815,7 @@ class ArchiveRecreater:
         target.save(comment=comment, additional_metadata={
             # keep some metadata as in original archive:
             'time': archive.metadata.time,
-            'time_end': archive.metadata.time_end,
+            'time_end': archive.metadata.get('time_end') or archive.metadata.time,
             'cmdline': archive.metadata.cmdline,
             # but also remember recreate metadata:
             'recreate_cmdline': sys.argv,

src/borg/archiver.py

@@ -40,7 +40,7 @@ from .archive import FilesystemObjectProcessors, MetadataCollector, ChunksProces
 from .cache import Cache, assert_secure
 from .constants import *  # NOQA
 from .compress import CompressionSpec
-from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
+from .crypto.key import key_creator, key_argument_names, tam_required_file, tam_required, RepoKey, PassphraseKey
 from .crypto.keymanager import KeyManager
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .helpers import Error, NoManifestError, set_ec
@@ -1326,7 +1326,7 @@ class Archiver:
             keep += prune_split(archives, '%Y', args.yearly, keep)
         to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints))
         stats = Statistics()
-        with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache:
+        with Cache(repository, key, manifest, do_files=False, lock_wait=self.lock_wait) as cache:
             list_logger = logging.getLogger('borg.output.list')
             if args.output_list:
                 # set up counters for the progress display
@@ -1959,6 +1959,8 @@ class Archiver:
         parser.print_help()
         return EXIT_SUCCESS
 
+    do_maincommand_help = do_subcommand_help
+
     def preprocess_args(self, args):
         deprecations = [
             # ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'),
@@ -2152,8 +2154,6 @@ class Archiver:
                               help='show/log the borg version')
             add_common_option('--show-rc', dest='show_rc', action='store_true',
                               help='show/log the return code (rc)')
-            add_common_option('--no-files-cache', dest='cache_files', action='store_false',
-                              help='do not load/update the file metadata cache used to detect unchanged files')
             add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT,
                               help='set umask to M (local and remote, default: %(default)04o)')
             add_common_option('--remote-path', metavar='PATH', dest='remote_path',
@@ -2226,6 +2226,7 @@ class Archiver:
 
         parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups',
                                          add_help=False)
+        parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser))
         parser.common_options = self.CommonOptions(define_common_options,
                                                    suffix_precedence=('_maincommand', '_midcommand', '_subcommand'))
         parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__,
@@ -2383,7 +2384,7 @@ class Archiver:
                                type=location_validator(archive=False),
                                help='repository to create')
         subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True,
-                               choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'),
+                               choices=key_argument_names(),
                                help='select encryption key mode **(required)**')
         subparser.add_argument('--append-only', dest='append_only', action='store_true',
                                help='create an append-only mode repository')
@@ -2723,6 +2724,8 @@ class Archiver:
                                help='output stats as JSON. Implies ``--stats``.')
         subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true',
                                help='experimental: do not synchronize the cache. Implies not using the files cache.')
+        subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
+                               help='do not load/update the file metadata cache used to detect unchanged files')
 
         define_exclusion_group(subparser, tag_files=True)
 

src/borg/cache.py

@@ -85,11 +85,11 @@ class SecurityManager:
         logger.debug('security: current location   %s', current_location)
         logger.debug('security: key type           %s', str(key.TYPE))
         logger.debug('security: manifest timestamp %s', manifest.timestamp)
-        with open(self.location_file, 'w') as fd:
+        with SaveFile(self.location_file) as fd:
             fd.write(current_location)
-        with open(self.key_type_file, 'w') as fd:
+        with SaveFile(self.key_type_file) as fd:
             fd.write(str(key.TYPE))
-        with open(self.manifest_ts_file, 'w') as fd:
+        with SaveFile(self.manifest_ts_file) as fd:
             fd.write(manifest.timestamp)
 
     def assert_location_matches(self, cache_config=None):
@@ -119,7 +119,7 @@ class SecurityManager:
                 raise Cache.RepositoryAccessAborted()
             # adapt on-disk config immediately if the new location was accepted
             logger.debug('security: updating location stored in cache and security dir')
-            with open(self.location_file, 'w') as fd:
+            with SaveFile(self.location_file) as fd:
                 fd.write(repository_location)
             if cache_config:
                 cache_config.save()
@@ -470,7 +470,7 @@ class LocalCache(CacheStatsMixin):
         self.cache_config.create()
         ChunkIndex().write(os.path.join(self.path, 'chunks'))
         os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
-        with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
+        with SaveFile(os.path.join(self.path, 'files'), binary=True):
             pass  # empty file
 
     def _do_open(self):
@@ -844,7 +844,7 @@ class LocalCache(CacheStatsMixin):
             shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
             os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         self.chunks = ChunkIndex()
-        with open(os.path.join(self.path, 'files'), 'wb'):
+        with SaveFile(os.path.join(self.path, 'files'), binary=True):
             pass  # empty file
         self.cache_config.manifest_id = ''
         self.cache_config._config.set('cache', 'manifest', '')

src/borg/compress.pyx

@@ -246,7 +246,7 @@ class Auto(CompressorBase):
         lz4_data = self.lz4.compress(data)
         ratio = len(lz4_data) / len(data)
         if ratio < 0.97:
-            return self.compressor, None
+            return self.compressor, lz4_data
         elif ratio < 1:
             return self.lz4, lz4_data
         else:
@@ -257,9 +257,24 @@ class Auto(CompressorBase):
 
     def compress(self, data):
         compressor, lz4_data = self._decide(data)
-        if lz4_data is None:
-            return compressor.compress(data)
+        if compressor is self.lz4:
+            # we know that trying to compress with expensive compressor is likely pointless,
+            # but lz4 managed to at least squeeze the data a bit.
+            return lz4_data
+        if compressor is self.none:
+            # we know that trying to compress with expensive compressor is likely pointless
+            # and also lz4 did not manage to squeeze the data (not even a bit).
+            uncompressed_data = compressor.compress(data)
+            return uncompressed_data
+        # if we get here, the decider decided to try the expensive compressor.
+        # we also know that lz4_data is smaller than uncompressed data.
+        exp_compressed_data = compressor.compress(data)
+        ratio = len(exp_compressed_data) / len(lz4_data)
+        if ratio < 0.99:
+            # the expensive compressor managed to squeeze the data significantly better than lz4.
+            return exp_compressed_data
         else:
+            # otherwise let's just store the lz4 data, which decompresses extremely fast.
             return lz4_data
 
     def decompress(self, data):

src/borg/crypto/key.py

@@ -103,11 +103,16 @@ class KeyBlobStorage:
 def key_creator(repository, args):
     for key in AVAILABLE_KEY_TYPES:
         if key.ARG_NAME == args.encryption:
+            assert key.ARG_NAME is not None
             return key.create(repository, args)
     else:
         raise ValueError('Invalid encryption mode "%s"' % args.encryption)
 
 
+def key_argument_names():
+    return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME]
+
+
 def identify_key(manifest_data):
     key_type = manifest_data[0]
     if key_type == PassphraseKey.TYPE:

src/borg/helpers/fs.py

@@ -1,3 +1,4 @@
+import errno
 import os
 import os.path
 import re
@@ -141,8 +142,13 @@ def truncate_and_unlink(path):
     recover. Refer to the "File system interaction" section
     in repository.py for further explanations.
     """
-    with open(path, 'r+b') as fd:
-        fd.truncate()
+    try:
+        with open(path, 'r+b') as fd:
+            fd.truncate()
+    except OSError as err:
+        if err.errno != errno.ENOTSUP:
+            raise
+        # don't crash if the above ops are not supported.
     os.unlink(path)
 
 

src/borg/logger.py

@@ -80,6 +80,8 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
                 logging.config.fileConfig(f)
             configured = True
             logger = logging.getLogger(__name__)
+            borg_logger = logging.getLogger('borg')
+            borg_logger.json = json
             logger.debug('using logging configuration read from "{0}"'.format(conf_fname))
             warnings.showwarning = _log_warning
             return None

src/borg/platform/linux.pyx

@@ -72,8 +72,11 @@ BSD_TO_LINUX_FLAGS = {
 
 
 def set_flags(path, bsd_flags, fd=None):
-    if fd is None and stat.S_ISLNK(os.lstat(path).st_mode):
-        return
+    if fd is None:
+        st = os.stat(path, follow_symlinks=False)
+        if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
+            # see comment in get_flags()
+            return
     cdef int flags = 0
     for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items():
         if bsd_flags & bsd_flag:
@@ -92,6 +95,10 @@ def set_flags(path, bsd_flags, fd=None):
 
 
 def get_flags(path, st):
+    if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode):
+        # avoid opening devices files - trying to open non-present devices can be rather slow.
+        # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway.
+        return 0
     cdef int linux_flags
     try:
         fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW)

src/borg/repository.py

@@ -266,7 +266,7 @@ class Repository:
             try:
                 os.link(config_path, old_config_path)
             except OSError as e:
-                if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM):
+                if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM, errno.ENOTSUP):
                     logger.warning("Hardlink failed, cannot securely erase old config file")
                 else:
                     raise

src/borg/testsuite/compress.py

@@ -110,12 +110,18 @@ def test_compressor():
 
 
 def test_auto():
-    compressor = CompressionSpec('auto,zlib,9').compressor
-
-    compressed = compressor.compress(bytes(500))
-    assert Compressor.detect(compressed) == ZLIB
-
-    compressed = compressor.compress(b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~')
+    compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor
+    compressor_lz4 = CompressionSpec('lz4').compressor
+    compressor_zlib = CompressionSpec('zlib,9').compressor
+    data = bytes(500)
+    compressed_auto_zlib = compressor_auto_zlib.compress(data)
+    compressed_lz4 = compressor_lz4.compress(data)
+    compressed_zlib = compressor_zlib.compress(data)
+    ratio = len(compressed_zlib) / len(compressed_lz4)
+    assert Compressor.detect(compressed_auto_zlib) == ZLIB if ratio < 0.99 else LZ4
+
+    data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~'
+    compressed = compressor_auto_zlib.compress(data)
     assert Compressor.detect(compressed) == CNONE