Merge pull request #1955 from ThomasWaldmann/merge-1.0-maint

merge 1.0-maint

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,8 @@
+Thank you for contributing code to Borg, your help is appreciated!
+
+Please, before you submit a pull request, make sure it complies with the
+guidelines given in our documentation:
+
+https://borgbackup.readthedocs.io/en/latest/development.html#contributions
+
+**Please remove all above text before submitting your pull request.**

README.rst

@@ -75,6 +75,7 @@ Main features
     * FreeBSD
     * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet)
     * Cygwin (not supported, no binaries yet)
+    * Linux Subsystem of Windows 10 (not supported)
 
 **Free and Open Source Software**
   * security and functionality can be audited independently

conftest.py

@@ -2,6 +2,9 @@ import os
 
 import pytest
 
+# needed to get pretty assertion failures in unit tests:
+pytest.register_assert_rewrite('borg.testsuite')
+
 from borg.logger import setup_logging
 
 # Ensure that the loggers exist for all tests

docs/changes.rst

@@ -71,6 +71,39 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
+Version 1.0.9 (not released yet)
+--------------------------------
+
+Bug fixes:
+
+- borg check:
+
+  - rebuild manifest if it's corrupted
+  - skip corrupted chunks during manifest rebuild
+- fix TypeError in integrity error handler, #1903, #1894
+- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930
+- fix wrong duration/timestamps if system clock jumped during a create
+- fix progress display not updating if system clock jumps backwards
+- fix checkpoint interval being incorrect if system clock jumps
+
+Other changes:
+
+- docs:
+
+  - add python3-devel as a dependency for cygwin-based installation
+  - clarify extract is relative to current directory
+  - FAQ: fix link to changelog
+  - markup fixes
+- tests:
+
+  - test_get_(cache|keys)_dir: clean env state, #1897
+  - get back pytest's pretty assertion failures, #1938
+- setup.py build_usage:
+
+  - fixed build_usage not processing all commands
+  - fixed build_usage not generating includes for debug commands
+
+
 Version 1.0.9rc1 (2016-11-27)
 -----------------------------
 

docs/installation.rst

@@ -290,12 +290,21 @@ and commands to make fuse work for using the mount command.
      sysctl vfs.usermount=1
 
 
+Windows 10's Linux Subsystem
+++++++++++++++++++++++++++++
+
+.. note::
+    Running under Windows 10's Linux Subsystem is experimental and has not been tested much yet.
+
+Just follow the Ubuntu Linux installation steps. You can omit the FUSE stuff, it won't work anyway.
+
+
 Cygwin
 ++++++
 
 .. note::
     Running under Cygwin is experimental and has only been tested with Cygwin
-    (x86-64) v2.5.2.
+    (x86-64) v2.5.2. Remote repositories are known broken, local repositories should work.
 
 Use the Cygwin installer to install the dependencies::
 

docs/usage/debug_delete-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_delete-obj:
+
+borg debug delete-obj
+---------------------
+::
+
+    borg debug delete-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to delete from the repo
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command deletes objects from the repository.

docs/usage/debug_dump-archive-items.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-archive-items:
+
+borg debug dump-archive-items
+-----------------------------
+::
+
+    borg debug dump-archive-items <options> ARCHIVE
+
+positional arguments
+    ARCHIVE
+        archive to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.

docs/usage/debug_dump-repo-objs.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-repo-objs:
+
+borg debug dump-repo-objs
+-------------------------
+::
+
+    borg debug dump-repo-objs <options> REPOSITORY
+
+positional arguments
+    REPOSITORY
+        repo to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) repo objects to files.

docs/usage/debug_get-obj.rst.inc

@@ -0,0 +1,25 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_get-obj:
+
+borg debug get-obj
+------------------
+::
+
+    borg debug get-obj <options> REPOSITORY ID PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    ID
+        hex object ID to get from the repo
+    PATH
+        file to write object data into
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command gets an object from the repository.

docs/usage/debug_info.rst.inc

@@ -0,0 +1,19 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_info:
+
+borg debug info
+---------------
+::
+
+    borg debug info <options>
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays some system information that might be useful for bug
+reports and debugging problems. If a traceback happens, this information is
+already appended at the end of the traceback.

docs/usage/debug_put-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_put-obj:
+
+borg debug put-obj
+------------------
+::
+
+    borg debug put-obj <options> REPOSITORY PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    PATH
+        file(s) to read and create object(s) from
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command puts objects into the repository.

docs/usage/debug_refcount-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_refcount-obj:
+
+borg debug refcount-obj
+-----------------------
+::
+
+    borg debug refcount-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to show refcounts for
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays the reference count for objects from the repository.

docs/usage/list.rst.inc

@@ -95,3 +95,5 @@ The following keys are available for --format:
  - archiveid
  - archivename
  - extra: prepends {source} with " -> " for soft links and " link to " for hard links
+
+ - health: either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)

docs/usage/recreate.rst.inc

@@ -64,6 +64,8 @@ Description
 
 Recreate the contents of existing archives.
 
+This is an *experimental* feature. Do *not* use this on your only backup.
+
 --exclude, --exclude-from and PATH have the exact same semantics
 as in "borg create". If PATHs are specified the resulting archive
 will only contain files from these PATHs.
@@ -80,15 +82,6 @@ There is no risk of data loss by this.
 used to have upgraded Borg 0.xx or Attic archives deduplicate with
 Borg 1.x archives.
 
-borg recreate is signal safe. Send either SIGINT (Ctrl-C on most terminals) or
-SIGTERM to request termination.
-
-Use the *exact same* command line to resume the operation later - changing excludes
-or paths will lead to inconsistencies (changed excludes will only apply to newly
-processed files/dirs). Changing compression leads to incorrect size information
-(which does not cause any data loss, but can be misleading).
-Changing chunker params between invocations might lead to data loss.
-
 USE WITH CAUTION.
 Depending on the PATHs and patterns given, recreate can be used to permanently
 delete files from archives.
@@ -103,5 +96,5 @@ With --target the original archive is not replaced, instead a new archive is cre
 
 When rechunking space usage can be substantial, expect at least the entire
 deduplicated size of the archives using the previous chunker params.
-When recompressing approximately 1 % of the repository size or 512 MB
-(whichever is greater) of additional space is used.
+When recompressing expect approx. (throughput / checkpoint-interval) in space usage,
+assuming all chunks are recompressed.

docs/usage/serve.rst.inc

@@ -10,7 +10,7 @@ borg serve
 
 optional arguments
     ``--restrict-to-path PATH``
-        | restrict repository access to PATH
+        | restrict repository access to PATH. Can be specified multiple times to allow the client access to several directories. Access to all sub-directories is granted implicitly; PATH doesn't need to directly point to a repository.
     ``--append-only``
         | only allow appending to repository segment files
 

setup.py

@@ -226,11 +226,14 @@ class build_usage(Command):
             return
         print('found commands: %s' % list(choices.keys()))
 
-        for command, parser in choices.items():
+        for command, parser in sorted(choices.items()):
+            if command.startswith('debug'):
+                print('skipping', command)
+                continue
             print('generating help for %s' % command)
 
             if self.generate_level(command + " ", parser, Archiver):
-                break
+                continue
 
             with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
                 doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")

src/borg/archive.py

@@ -5,7 +5,7 @@ import stat
 import sys
 import time
 from contextlib import contextmanager
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from functools import partial
 from getpass import getuser
 from io import BytesIO
@@ -81,7 +81,7 @@ class Statistics:
         return format_file_size(self.csize)
 
     def show_progress(self, item=None, final=False, stream=None, dt=None):
-        now = time.time()
+        now = time.monotonic()
         if dt is None or now - self.last_progress > dt:
             self.last_progress = now
             columns, lines = get_terminal_size()
@@ -255,7 +255,7 @@ class Archive:
 
     def __init__(self, repository, key, manifest, name, cache=None, create=False,
                  checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False,
-                 chunker_params=CHUNKER_PARAMS, start=None, end=None, compression=None, compression_files=None,
+                 chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, compression=None, compression_files=None,
                  consider_part_files=False):
         self.cwd = os.getcwd()
         self.key = key
@@ -270,10 +270,13 @@ class Archive:
         self.numeric_owner = numeric_owner
         self.noatime = noatime
         self.noctime = noctime
+        assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
         if start is None:
             start = datetime.utcnow()
+            start_monotonic = time.monotonic()
         self.chunker_params = chunker_params
         self.start = start
+        self.start_monotonic = start_monotonic
         if end is None:
             end = datetime.utcnow()
         self.end = end
@@ -288,7 +291,7 @@ class Archive:
             key.compression_decider2 = CompressionDecider2(compression or CompressionSpec('none'))
             if name in manifest.archives:
                 raise self.AlreadyExists(name)
-            self.last_checkpoint = time.time()
+            self.last_checkpoint = time.monotonic()
             i = 0
             while True:
                 self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')
@@ -381,14 +384,17 @@ Number of files: {0.stats.nfiles}'''.format(
         if name in self.manifest.archives:
             raise self.AlreadyExists(name)
         self.items_buffer.flush(flush=True)
+        duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
         if timestamp is None:
             self.end = datetime.utcnow()
+            self.start = self.end - duration
             start = self.start
             end = self.end
         else:
             self.end = timestamp
-            start = timestamp
-            end = timestamp  # we only have 1 value
+            self.start = timestamp - duration
+            end = timestamp
+            start = self.start
         metadata = {
             'version': 1,
             'name': name,
@@ -787,9 +793,9 @@ Number of files: {0.stats.nfiles}'''.format(
             item.chunks.append(chunk_processor(data))
             if self.show_progress:
                 self.stats.show_progress(item=item, dt=0.2)
-            if self.checkpoint_interval and time.time() - self.last_checkpoint > self.checkpoint_interval:
+            if self.checkpoint_interval and time.monotonic() - self.last_checkpoint > self.checkpoint_interval:
                 from_chunk, part_number = self.write_part_file(item, from_chunk, part_number)
-                self.last_checkpoint = time.time()
+                self.last_checkpoint = time.monotonic()
         else:
             if part_number > 1:
                 if item.chunks[from_chunk:]:
@@ -797,7 +803,7 @@ Number of files: {0.stats.nfiles}'''.format(
                     # chunks (if any) into a part item also (so all parts can be concatenated to get
                     # the complete file):
                     from_chunk, part_number = self.write_part_file(item, from_chunk, part_number)
-                    self.last_checkpoint = time.time()
+                    self.last_checkpoint = time.monotonic()
 
                 # if we created part files, we have referenced all chunks from the part files,
                 # but we also will reference the same chunks also from the final, complete file:

src/borg/archiver.py

@@ -12,11 +12,11 @@ import stat
 import subprocess
 import sys
 import textwrap
+import time
 import traceback
 from binascii import unhexlify
 from datetime import datetime
 from itertools import zip_longest
-from operator import attrgetter
 
 from .logger import create_logger, setup_logging
 logger = create_logger()
@@ -327,7 +327,6 @@ class Archiver:
                 if args.progress:
                     archive.stats.show_progress(final=True)
                 if args.stats:
-                    archive.end = datetime.utcnow()
                     log_multi(DASHES,
                               str(archive),
                               DASHES,
@@ -341,6 +340,7 @@ class Archiver:
         self.ignore_inode = args.ignore_inode
         dry_run = args.dry_run
         t0 = datetime.utcnow()
+        t0_monotonic = time.monotonic()
         if not dry_run:
             with Cache(repository, key, manifest, do_files=args.cache_files, progress=args.progress,
                        lock_wait=self.lock_wait) as cache:
@@ -348,7 +348,7 @@ class Archiver:
                                   create=True, checkpoint_interval=args.checkpoint_interval,
                                   numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime,
                                   progress=args.progress,
-                                  chunker_params=args.chunker_params, start=t0,
+                                  chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic,
                                   compression=args.compression, compression_files=args.compression_files)
                 create_inner(archive, cache)
         else:
@@ -1485,6 +1485,11 @@ class Archiver:
             parser.error('No help available on %s' % (args.topic,))
         return self.exit_code
 
+    def do_subcommand_help(self, parser, args):
+        """display infos about subcommand"""
+        parser.print_help()
+        return EXIT_SUCCESS
+
     def preprocess_args(self, args):
         deprecations = [
             # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),
@@ -1723,13 +1728,14 @@ class Archiver:
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
-        subparser = subparsers.add_parser('key', add_help=True,
+        subparser = subparsers.add_parser('key', parents=[common_parser], add_help=False,
                                           description="Manage a keyfile or repokey of a repository",
                                           epilog="",
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='manage repository key')
 
         key_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         key_export_epilog = textwrap.dedent("""
         If repository encryption is used, the repository is inaccessible
@@ -2512,13 +2518,14 @@ class Archiver:
         in case you ever run into some severe malfunction. Use them only if you know
         what you are doing or if a trusted developer tells you what to do.""")
 
-        subparser = subparsers.add_parser('debug', add_help=True,
+        subparser = subparsers.add_parser('debug', parents=[common_parser], add_help=False,
                                           description='debugging command (not intended for normal use)',
                                           epilog=debug_epilog,
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='debugging command (not intended for normal use)')
 
         debug_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         debug_info_epilog = textwrap.dedent("""
         This command displays some system information that might be useful for bug

src/borg/helpers.py

@@ -889,6 +889,17 @@ class Location:
     """
     proto = user = host = port = path = archive = None
 
+    # user must not contain "@", ":" or "/".
+    # Quoting adduser error message:
+    # "To avoid problems, the username should consist only of letters, digits,
+    # underscores, periods, at signs and dashes, and not start with a dash
+    # (as defined by IEEE Std 1003.1-2001)."
+    # We use "@" as separator between username and hostname, so we must
+    # disallow it within the pure username part.
+    optional_user_re = r"""
+        (?:(?P<user>[^@:/]+)@)?
+    """
+
     # path must not contain :: (it ends at :: or string end), but may contain single colons.
     # to avoid ambiguities with other regexes, it must also not start with ":".
     path_re = r"""
@@ -907,7 +918,7 @@ class Location:
     # regexes for misc. kinds of supported location specifiers:
     ssh_re = re.compile(r"""
         (?P<proto>ssh)://                                   # ssh://
-        (?:(?P<user>[^@]+)@)?                               # user@  (optional)
+        """ + optional_user_re + r"""                       # user@  (optional)
         (?P<host>[^:/]+)(?::(?P<port>\d+))?                 # host or host:port
         """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive
 
@@ -918,7 +929,7 @@ class Location:
     # note: scp_re is also use for local pathes
     scp_re = re.compile(r"""
         (
-            (?:(?P<user>[^@]+)@)?                           # user@  (optional)
+            """ + optional_user_re + r"""                   # user@  (optional)
             (?P<host>[^:/]+):                               # host: (don't match / in host to disambiguate from file:)
         )?                                                  # user@host: part is optional
         """ + path_re + optional_archive_re, re.VERBOSE)    # path with optional archive

src/borg/testsuite/archiver.py

@@ -328,7 +328,8 @@ class ArchiverTestCaseBase(BaseTestCase):
         except PermissionError:
             have_root = False
         except OSError as e:
-            if e.errno != errno.EINVAL:
+            # Note: ENOSYS "Function not implemented" happens as non-root on Win 10 Linux Subsystem.
+            if e.errno not in (errno.EINVAL, errno.ENOSYS):
                 raise
             have_root = False
         return have_root

src/borg/testsuite/helpers.py

@@ -97,6 +97,13 @@ class TestLocationWithoutEnv:
         assert repr(Location('/abs/path:with:colons')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)"
 
+    def test_user_parsing(self):
+        # see issue #1930
+        assert repr(Location('host:path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='path', archive='2016-12-31@23:59:59')"
+        assert repr(Location('ssh://host/path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='/path', archive='2016-12-31@23:59:59')"
+
     def test_underspecified(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
         with pytest.raises(ValueError):