Merge branch 'merge-1.0-maint' of https://github.com/ThomasWaldmann/borg into ThomasWaldmann-merge-1.0-maint

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,8 @@
+Thank you for contributing code to Borg, your help is appreciated!
+
+Please, before you submit a pull request, make sure it complies with the
+guidelines given in our documentation:
+
+https://borgbackup.readthedocs.io/en/latest/development.html#contributions
+
+**Please remove all above text before submitting your pull request.**

conftest.py

@@ -2,6 +2,9 @@ import os
 
 import pytest
 
+# needed to get pretty assertion failures in unit tests:
+pytest.register_assert_rewrite('borg')
+
 from borg.logger import setup_logging
 
 # Ensure that the loggers exist for all tests

docs/changes.rst

@@ -71,6 +71,36 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
+Version 1.0.9 (not released yet)
+--------------------------------
+
+Bug fixes:
+
+- borg check:
+
+  - rebuild manifest if it's corrupted
+  - skip corrupted chunks during manifest rebuild
+- fix TypeError in integrity error handler, #1903, #1894
+- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930
+
+Other changes:
+
+- docs:
+
+  - add python3-devel as a dependency for cygwin-based installation
+  - clarify extract is relative to current directory
+  - FAQ: fix link to changelog
+  - markup fixes
+- tests:
+
+  - test_get_(cache|keys)_dir: clean env state, #1897
+  - get back pytest's pretty assertion failures, #1938
+- setup.py build_usage:
+
+  - fixed build_usage not processing all commands
+  - fixed build_usage not generating includes for debug commands
+
+
 Version 1.0.9rc1 (2016-11-27)
 -----------------------------
 

docs/usage/debug_delete-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_delete-obj:
+
+borg debug delete-obj
+---------------------
+::
+
+    borg debug delete-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to delete from the repo
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command deletes objects from the repository.

docs/usage/debug_dump-archive-items.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-archive-items:
+
+borg debug dump-archive-items
+-----------------------------
+::
+
+    borg debug dump-archive-items <options> ARCHIVE
+
+positional arguments
+    ARCHIVE
+        archive to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.

docs/usage/debug_dump-repo-objs.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-repo-objs:
+
+borg debug dump-repo-objs
+-------------------------
+::
+
+    borg debug dump-repo-objs <options> REPOSITORY
+
+positional arguments
+    REPOSITORY
+        repo to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) repo objects to files.

docs/usage/debug_get-obj.rst.inc

@@ -0,0 +1,25 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_get-obj:
+
+borg debug get-obj
+------------------
+::
+
+    borg debug get-obj <options> REPOSITORY ID PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    ID
+        hex object ID to get from the repo
+    PATH
+        file to write object data into
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command gets an object from the repository.

docs/usage/debug_info.rst.inc

@@ -0,0 +1,19 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_info:
+
+borg debug info
+---------------
+::
+
+    borg debug info <options>
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays some system information that might be useful for bug
+reports and debugging problems. If a traceback happens, this information is
+already appended at the end of the traceback.

docs/usage/debug_put-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_put-obj:
+
+borg debug put-obj
+------------------
+::
+
+    borg debug put-obj <options> REPOSITORY PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    PATH
+        file(s) to read and create object(s) from
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command puts objects into the repository.

docs/usage/debug_refcount-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_refcount-obj:
+
+borg debug refcount-obj
+-----------------------
+::
+
+    borg debug refcount-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to show refcounts for
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays the reference count for objects from the repository.

docs/usage/list.rst.inc

@@ -95,3 +95,5 @@ The following keys are available for --format:
  - archiveid
  - archivename
  - extra: prepends {source} with " -> " for soft links and " link to " for hard links
+
+ - health: either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)

docs/usage/recreate.rst.inc

@@ -64,6 +64,8 @@ Description
 
 Recreate the contents of existing archives.
 
+This is an *experimental* feature. Do *not* use this on your only backup.
+
 --exclude, --exclude-from and PATH have the exact same semantics
 as in "borg create". If PATHs are specified the resulting archive
 will only contain files from these PATHs.
@@ -80,15 +82,6 @@ There is no risk of data loss by this.
 used to have upgraded Borg 0.xx or Attic archives deduplicate with
 Borg 1.x archives.
 
-borg recreate is signal safe. Send either SIGINT (Ctrl-C on most terminals) or
-SIGTERM to request termination.
-
-Use the *exact same* command line to resume the operation later - changing excludes
-or paths will lead to inconsistencies (changed excludes will only apply to newly
-processed files/dirs). Changing compression leads to incorrect size information
-(which does not cause any data loss, but can be misleading).
-Changing chunker params between invocations might lead to data loss.
-
 USE WITH CAUTION.
 Depending on the PATHs and patterns given, recreate can be used to permanently
 delete files from archives.
@@ -103,5 +96,5 @@ With --target the original archive is not replaced, instead a new archive is cre
 
 When rechunking space usage can be substantial, expect at least the entire
 deduplicated size of the archives using the previous chunker params.
-When recompressing approximately 1 % of the repository size or 512 MB
-(whichever is greater) of additional space is used.
+When recompressing expect approx. (throughput / checkpoint-interval) in space usage,
+assuming all chunks are recompressed.

docs/usage/serve.rst.inc

@@ -10,7 +10,7 @@ borg serve
 
 optional arguments
     ``--restrict-to-path PATH``
-        | restrict repository access to PATH
+        | restrict repository access to PATH. Can be specified multiple times to allow the client access to several directories. Access to all sub-directories is granted implicitly; PATH doesn't need to directly point to a repository.
     ``--append-only``
         | only allow appending to repository segment files
 

setup.py

@@ -226,11 +226,14 @@ class build_usage(Command):
             return
         print('found commands: %s' % list(choices.keys()))
 
-        for command, parser in choices.items():
+        for command, parser in sorted(choices.items()):
+            if command.startswith('debug'):
+                print('skipping', command)
+                continue
             print('generating help for %s' % command)
 
             if self.generate_level(command + " ", parser, Archiver):
-                break
+                continue
 
             with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
                 doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")

src/borg/archiver.py

@@ -1485,6 +1485,11 @@ class Archiver:
             parser.error('No help available on %s' % (args.topic,))
         return self.exit_code
 
+    def do_subcommand_help(self, parser, args):
+        """display infos about subcommand"""
+        parser.print_help()
+        return EXIT_SUCCESS
+
     def preprocess_args(self, args):
         deprecations = [
             # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),
@@ -1723,13 +1728,14 @@ class Archiver:
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
-        subparser = subparsers.add_parser('key', add_help=True,
+        subparser = subparsers.add_parser('key', parents=[common_parser], add_help=False,
                                           description="Manage a keyfile or repokey of a repository",
                                           epilog="",
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='manage repository key')
 
         key_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         key_export_epilog = textwrap.dedent("""
         If repository encryption is used, the repository is inaccessible
@@ -2512,13 +2518,14 @@ class Archiver:
         in case you ever run into some severe malfunction. Use them only if you know
         what you are doing or if a trusted developer tells you what to do.""")
 
-        subparser = subparsers.add_parser('debug', add_help=True,
+        subparser = subparsers.add_parser('debug', parents=[common_parser], add_help=False,
                                           description='debugging command (not intended for normal use)',
                                           epilog=debug_epilog,
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='debugging command (not intended for normal use)')
 
         debug_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         debug_info_epilog = textwrap.dedent("""
         This command displays some system information that might be useful for bug

src/borg/helpers.py

@@ -889,6 +889,17 @@ class Location:
     """
     proto = user = host = port = path = archive = None
 
+    # user must not contain "@", ":" or "/".
+    # Quoting adduser error message:
+    # "To avoid problems, the username should consist only of letters, digits,
+    # underscores, periods, at signs and dashes, and not start with a dash
+    # (as defined by IEEE Std 1003.1-2001)."
+    # We use "@" as separator between username and hostname, so we must
+    # disallow it within the pure username part.
+    optional_user_re = r"""
+        (?:(?P<user>[^@:/]+)@)?
+    """
+
     # path must not contain :: (it ends at :: or string end), but may contain single colons.
     # to avoid ambiguities with other regexes, it must also not start with ":".
     path_re = r"""
@@ -907,7 +918,7 @@ class Location:
     # regexes for misc. kinds of supported location specifiers:
     ssh_re = re.compile(r"""
         (?P<proto>ssh)://                                   # ssh://
-        (?:(?P<user>[^@]+)@)?                               # user@  (optional)
+        """ + optional_user_re + r"""                       # user@  (optional)
         (?P<host>[^:/]+)(?::(?P<port>\d+))?                 # host or host:port
         """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive
 
@@ -918,7 +929,7 @@ class Location:
     # note: scp_re is also use for local pathes
     scp_re = re.compile(r"""
         (
-            (?:(?P<user>[^@]+)@)?                           # user@  (optional)
+            """ + optional_user_re + r"""                   # user@  (optional)
             (?P<host>[^:/]+):                               # host: (don't match / in host to disambiguate from file:)
         )?                                                  # user@host: part is optional
         """ + path_re + optional_archive_re, re.VERBOSE)    # path with optional archive

src/borg/testsuite/helpers.py

@@ -97,6 +97,13 @@ class TestLocationWithoutEnv:
         assert repr(Location('/abs/path:with:colons')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)"
 
+    def test_user_parsing(self):
+        # see issue #1930
+        assert repr(Location('host:path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='path', archive='2016-12-31@23:59:59')"
+        assert repr(Location('ssh://host/path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='/path', archive='2016-12-31@23:59:59')"
+
     def test_underspecified(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
         with pytest.raises(ValueError):