Merge pull request #4572 from borgbackup/rel1110

release 1.1.10

Vagrantfile

@@ -63,9 +63,9 @@ def packages_darwin
     sudo softwareupdate --ignore "Install macOS High Sierra"
     sudo softwareupdate --install --all
     # get osxfuse 3.x release code from github:
-    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.8.2/osxfuse-3.8.2.dmg >osxfuse.dmg
+    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.8.3/osxfuse-3.8.3.dmg >osxfuse.dmg
     MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
-    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.8.2.pkg" -target /
+    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.8.3.pkg" -target /
     sudo chown -R vagrant /usr/local  # brew must be able to create stuff here
     ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     brew update
@@ -229,7 +229,7 @@ def install_pythons(boxname)
     pyenv install 3.4.0  # tests
     pyenv install 3.5.0  # tests
     pyenv install 3.6.0  # tests
-    pyenv install 3.5.6  # binary build, use latest 3.5.x release
+    pyenv install 3.5.7  # binary build, use latest 3.5.x release
     pyenv rehash
   EOF
 end
@@ -247,8 +247,8 @@ def build_pyenv_venv(boxname)
     . ~/.bash_profile
     cd /vagrant/borg
     # use the latest 3.5 release
-    pyenv global 3.5.6
-    pyenv virtualenv 3.5.6 borg-env
+    pyenv global 3.5.7
+    pyenv virtualenv 3.5.7 borg-env
     ln -s ~/.pyenv/versions/borg-env .
   EOF
 end

docs/changes.rst

@@ -171,8 +171,8 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
-Version 1.1.10 (not released yet)
----------------------------------
+Version 1.1.10 (2019-05-16)
+---------------------------
 
 Compatibility notes:
 
@@ -208,11 +208,19 @@ Fixes:
 
 New features:
 
+- bundle latest supported msgpack-python release (0.5.6), remove msgpack-python
+  from setup.py install_requires - by default we use the bundled code now.
+  optionally, we still support using an external msgpack (see hints in
+  setup.py), but this requires solid requirements management within
+  distributions and is not recommended.
+  borgbackup will break if you upgrade msgpack to an unsupported version.
 - display msgpack version as part of sysinfo (e.g. in tracebacks)
 - timestamp for borg delete --info added, #4359
+- enable placeholder usage in --comment and --glob-archives, #4559, #4495
 
 Other:
 
+- serve: do not check python/libc for borg serve, #4483
 - shell completions: borg diff second archive
 - release scripts: signing binaries with Qubes OS support
 - testing:
@@ -220,6 +228,9 @@ Other:
   - vagrant: upgrade openbsd box to 6.4
   - travis-ci: lock test env to py 3.4 compatible versions, #4343
   - get rid of confusing coverage warning, #2069
+  - rename test_mount_hardlinks to test_fuse_mount_hardlinks,
+    so both can be excluded by "not test_fuse".
+  - pure-py msgpack warning shall not make a lot of tests fail, #4558
 - docs:
 
   - add "SSH Configuration" section to "borg serve", #3988, #636, #4485
@@ -228,6 +239,11 @@ Other:
   - elaborate on append-only mode docs, #3504
   - installation: added Alpine Linux to distribution list, #4415
   - usage.html: only modify window.location when redirecting, #4133
+  - add msgpack license to docs/3rd_party/msgpack
+- vagrant / binary builds:
+
+  - use python 3.5.7 for builds
+  - use osxfuse 3.8.3
 
 
 Version 1.1.9 (2019-02-10)

docs/man/borg-benchmark-crud.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-BENCHMARK-CRUD 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-BENCHMARK-CRUD 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives.
 .

docs/man/borg-benchmark.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-BENCHMARK 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-BENCHMARK 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-benchmark \- benchmark command
 .

docs/man/borg-break-lock.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-BREAK-LOCK 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-BREAK-LOCK 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg.
 .

docs/man/borg-change-passphrase.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CHANGE-PASSPHRASE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-CHANGE-PASSPHRASE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-change-passphrase \- Change repository key file passphrase
 .

docs/man/borg-check.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CHECK 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-CHECK 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-check \- Check repository consistency
 .

docs/man/borg-common.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-COMMON 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-COMMON 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-common \- Common options of Borg commands
 .

docs/man/borg-compression.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-COMPRESSION 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-COMPRESSION 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-compression \- Details regarding compression
 .

docs/man/borg-config.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CONFIG 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-CONFIG 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-config \- get, set, and delete values in a repository or cache config file
 .

docs/man/borg-create.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CREATE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-CREATE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-create \- Create new archive
 .

docs/man/borg-delete.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-DELETE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-DELETE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-delete \- Delete an existing repository or archives
 .

docs/man/borg-diff.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-DIFF 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-DIFF 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-diff \- Diff contents of two archives
 .

docs/man/borg-export-tar.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-EXPORT-TAR 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-EXPORT-TAR 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-export-tar \- Export archive contents as a tarball
 .

docs/man/borg-extract.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-EXTRACT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-EXTRACT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-extract \- Extract archive contents
 .

docs/man/borg-info.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-INFO 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-INFO 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-info \- Show archive details such as disk space used
 .

docs/man/borg-init.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-INIT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-INIT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-init \- Initialize an empty repository
 .

docs/man/borg-key-change-passphrase.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-key-change-passphrase \- Change repository key file passphrase
 .

docs/man/borg-key-export.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-EXPORT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-KEY-EXPORT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-key-export \- Export the repository key for backup
 .

docs/man/borg-key-import.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-IMPORT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-KEY-IMPORT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-key-import \- Import the repository key from backup
 .

docs/man/borg-key-migrate-to-repokey.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-key-migrate-to-repokey \- Migrate passphrase -> repokey
 .

docs/man/borg-key.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-KEY 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-key \- Manage a keyfile or repokey of a repository
 .

docs/man/borg-list.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-LIST 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-LIST 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-list \- List archive or repository contents
 .

docs/man/borg-mount.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-MOUNT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-MOUNT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-mount \- Mount archive or an entire repository as a FUSE filesystem
 .

docs/man/borg-patterns.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PATTERNS 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-PATTERNS 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-patterns \- Details regarding patterns
 .

docs/man/borg-placeholders.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PLACEHOLDERS 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-PLACEHOLDERS 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-placeholders \- Details regarding placeholders
 .
@@ -32,8 +32,8 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH DESCRIPTION
 .sp
-Repository (or Archive) URLs, \fB\-\-prefix\fP and \fB\-\-remote\-path\fP values support these
-placeholders:
+Repository (or Archive) URLs, \fB\-\-prefix\fP, \fB\-\-glob\-archives\fP, \fB\-\-comment\fP
+and \fB\-\-remote\-path\fP values support these placeholders:
 .INDENT 0.0
 .TP
 .B {hostname}

docs/man/borg-prune.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PRUNE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-PRUNE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-prune \- Prune repository archives according to specified rules
 .

docs/man/borg-recreate.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-RECREATE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-RECREATE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-recreate \- Re-create archives
 .

docs/man/borg-rename.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-RENAME 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-RENAME 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-rename \- Rename an existing archive
 .

docs/man/borg-serve.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-SERVE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-SERVE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-serve \- Start in server mode. This command is usually not used manually.
 .
@@ -102,6 +102,48 @@ has been introduced in v7.2. We recommend to use
 in this case.
 .UNINDENT
 .UNINDENT
+.SS SSH Configuration
+.sp
+\fBborg serve\fP\(aqs pipes (\fBstdin\fP/\fBstdout\fP/\fBstderr\fP) are connected to the \fBsshd\fP process on the server side. In the event that the SSH connection between \fBborg serve\fP and the client is disconnected or stuck abnormally (for example, due to a network outage), it can take a long time for \fBsshd\fP to notice the client is disconnected. In the meantime, \fBsshd\fP continues running, and as a result so does the \fBborg serve\fP process holding the lock on the repository. This can cause subsequent \fBborg\fP operations on the remote repository to fail with the error: \fBFailed to create/acquire the lock\fP\&.
+.sp
+In order to avoid this, it is recommended to perform the following additional SSH configuration:
+.sp
+Either in the client side\(aqs \fB~/.ssh/config\fP file, or in the client\(aqs \fB/etc/ssh/ssh_config\fP file:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+Host backupserver
+        ServerAliveInterval 10
+        ServerAliveCountMax 30
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+Replacing \fBbackupserver\fP with the hostname, FQDN or IP address of the borg server.
+.sp
+This will cause the client to send a keepalive to the server every 10 seconds. If 30 consecutive keepalives are sent without a response (a time of 300 seconds), the ssh client process will be terminated, causing the borg process to terminate gracefully.
+.sp
+On the server side\(aqs \fBsshd\fP configuration file (typically \fB/etc/ssh/sshd_config\fP):
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+ClientAliveInterval 10
+ClientAliveCountMax 30
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
+.sp
+This will cause the server to send a keep alive to the client every 10 seconds. If 30 consecutive keepalives are sent without a response (a time of 300 seconds), the server\(aqs sshd process will be terminated, causing the \fBborg serve\fP process to terminate gracefully and release the lock on the repository.
+.sp
+If you then run borg commands with \fB\-\-lock\-wait 600\fP, this gives sufficient time for the borg serve processes to terminate after the SSH connection is torn down after the 300 second wait for the keepalives to fail.
+.sp
+You may, of course, modify the timeout values demonstrated above to values that suit your environment and use case.
 .SH SEE ALSO
 .sp
 \fIborg\-common(1)\fP

docs/man/borg-umount.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-UMOUNT 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-UMOUNT 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-umount \- un-mount the FUSE filesystem
 .

docs/man/borg-upgrade.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-UPGRADE 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-UPGRADE 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-upgrade \- upgrade a repository from a previous version
 .

docs/man/borg-with-lock.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-WITH-LOCK 1 "2019-02-10" "" "borg backup tool"
+.TH BORG-WITH-LOCK 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borg-with-lock \- run a user specified command with the repository lock held
 .

docs/man/borg.1

@@ -430,6 +430,7 @@ use \fI\%fqdn@uniqueid\fP\&.
 .TP
 .B BORG_LOGGING_CONF
 When set, use the given filename as \fI\%INI\fP\-style logging configuration.
+A basic example conf can be found at \fBdocs/misc/logging.conf\fP\&.
 .TP
 .B BORG_RSH
 When set, use this command instead of \fBssh\fP\&. This can be used to specify ssh options, such as

docs/man/borgfs.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORGFS 1 "2019-02-10" "" "borg backup tool"
+.TH BORGFS 1 "2019-05-16" "" "borg backup tool"
 .SH NAME
 borgfs \- Mount archive or an entire repository as a FUSE filesystem
 .

docs/usage/help.rst.inc

@@ -178,8 +178,8 @@ borg help placeholders
 ~~~~~~~~~~~~~~~~~~~~~~
 
 
-Repository (or Archive) URLs, ``--prefix`` and ``--remote-path`` values support these
-placeholders:
+Repository (or Archive) URLs, ``--prefix``, ``--glob-archives``, ``--comment``
+and ``--remote-path`` values support these placeholders:
 
 {hostname}
     The (short) hostname of the machine.