Merge pull request #2348 from ThomasWaldmann/rel110b4

Release 1.1.0b4

Vagrantfile

@@ -355,7 +355,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "centos7_64" do |b|
     b.vm.box = "centos/7"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64")
@@ -367,6 +367,9 @@ Vagrant.configure(2) do |config|
 
   config.vm.define "centos6_32" do |b|
     b.vm.box = "centos6-32"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 1024
+    end
     b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32")
     b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32")
@@ -378,7 +381,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "centos6_64" do |b|
     b.vm.box = "centos6-64"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64")
@@ -391,7 +394,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "xenial64" do |b|
     b.vm.box = "ubuntu/xenial64"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64")
@@ -402,7 +405,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "trusty64" do |b|
     b.vm.box = "ubuntu/trusty64"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
@@ -413,7 +416,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "jessie64" do |b|
     b.vm.box = "debian/jessie64"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64")
@@ -423,6 +426,9 @@ Vagrant.configure(2) do |config|
 
   config.vm.define "wheezy32" do |b|
     b.vm.box = "boxcutter/debian7-i386"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 1024
+    end
     b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32")
@@ -492,7 +498,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "openbsd64" do |b|
     b.vm.box = "openbsd60-64"  # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
@@ -503,7 +509,7 @@ Vagrant.configure(2) do |config|
   config.vm.define "netbsd64" do |b|
     b.vm.box = "netbsd70-64"
     b.vm.provider :virtualbox do |v|
-      v.memory = 768
+      v.memory = 1536
     end
     b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64")

docs/changes.rst

@@ -128,8 +128,8 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
-Version 1.1.0b4 (not released yet)
-----------------------------------
+Version 1.1.0b4 (2017-03-27)
+----------------------------
 
 Compatibility notes:
 

docs/man/borg-break-lock.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-BREAK-LOCK 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-BREAK-LOCK 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg.
 .

docs/man/borg-change-passphrase.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-CHANGE-PASSPHRASE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-change-passphrase \- Change repository key file passphrase
 .

docs/man/borg-check.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CHECK 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-CHECK 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-check \- Check repository consistency
 .

docs/man/borg-common.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-COMMON 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-COMMON 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-common \- Common options of Borg commands
 .
@@ -54,6 +54,9 @@ enable debug output, work on log level DEBUG
 .BI \-\-debug\-topic \ TOPIC
 enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.
 .TP
+.B \-\-log\-json
+Output one JSON object per log line instead of formatted text.
+.TP
 .BI \-\-lock\-wait \ N
 wait for the lock, but max. N seconds (default: 1).
 .TP
@@ -70,7 +73,7 @@ do not load/update the file metadata cache used to detect unchanged files
 set umask to M (local and remote, default: 0077)
 .TP
 .BI \-\-remote\-path \ PATH
-set remote path to executable (default: "borg")
+use PATH as borg executable on the remote (default: "borg")
 .TP
 .BI \-\-remote\-ratelimit \ rate
 set remote network upload rate limit in kiByte/s (default: 0=unlimited)

docs/man/borg-compression.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-COMPRESSION 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-COMPRESSION 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-compression \- Details regarding compression
 .
@@ -32,21 +32,21 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH DESCRIPTION
 .sp
-Compression is off by default, if you want some, you have to specify what you want.
+Compression is lz4 by default. If you want something else, you have to specify what you want.
 .sp
 Valid compression specifiers are:
 .sp
 none
 .INDENT 0.0
 .INDENT 3.5
-Do not compress. (default)
+Do not compress.
 .UNINDENT
 .UNINDENT
 .sp
 lz4
 .INDENT 0.0
 .INDENT 3.5
-Use lz4 compression. High speed, low compression.
+Use lz4 compression. High speed, low compression. (default)
 .UNINDENT
 .UNINDENT
 .sp

docs/man/borg-create.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-CREATE 1 "2017-02-12" "" "borg backup tool"
+.TH BORG-CREATE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-create \- Create new archive
 .
@@ -36,9 +36,12 @@ borg create <options> ARCHIVE PATH
 .SH DESCRIPTION
 .sp
 This command creates a backup archive containing all files found while recursively
-traversing all paths specified. When giving \(aq\-\(aq as path, borg will read data
-from standard input and create a file \(aqstdin\(aq in the created archive from that
-data.
+traversing all paths specified. Paths are added to the archive as they are given,
+that means if relative paths are desired, the command has to be run from the correct
+directory.
+.sp
+When giving \(aq\-\(aq as path, borg will read data from standard input and create a
+file \(aqstdin\(aq in the created archive from that data.
 .sp
 The archive will consume almost no disk space for files or parts of files that
 have already been stored in other archives.
@@ -55,6 +58,11 @@ not provide correct inode information the \-\-ignore\-inode flag can be used. Th
 potentially decreases reliability of change detection, while avoiding always reading
 all files on these file systems.
 .sp
+The mount points of filesystems or filesystem snapshots should be the same for every
+creation of a new archive to ensure fast operation. This is because the file cache that
+is used to determine changed files quickly uses absolute filenames.
+If this is not possible, consider creating a bind mount to a stable location.
+.sp
 See the output of the "borg help patterns" command for more help on exclude patterns.
 See the output of the "borg help placeholders" command for more help on placeholders.
 .SH OPTIONS
@@ -86,6 +94,9 @@ output verbose list of items (files, dirs, ...)
 .TP
 .BI \-\-filter \ STATUSCHARS
 only display items with the given status characters
+.TP
+.B \-\-json
+output stats as JSON (implies \-\-stats)
 .UNINDENT
 .SS Exclusion options
 .INDENT 0.0
@@ -103,13 +114,19 @@ exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosauru
 exclude directories that are tagged by containing a filesystem object with the given NAME
 .TP
 .B \-\-keep\-exclude\-tags\fP,\fB  \-\-keep\-tag\-files
-keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise excluded caches/directories
+if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
+.TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
 .UNINDENT
 .SS Filesystem options
 .INDENT 0.0
 .TP
 .B \-x\fP,\fB  \-\-one\-file\-system
-stay in same file system, do not cross mount points
+stay in the same file system and do not store mount points of other file systems
 .TP
 .B \-\-numeric\-owner
 only store numeric user and group identifiers
@@ -175,7 +192,7 @@ $ borg create /path/to/repo::my\-files /home \e
     \-\-exclude \(aqsh:/home/*/.thumbnails\(aq
 
 # Backup the root filesystem into an archive named "root\-YYYY\-MM\-DD"
-# use zlib compression (good, but slow) \- default is no compression
+# use zlib compression (good, but slow) \- default is lz4 (fast, low compression ratio)
 $ borg create \-C zlib,6 /path/to/repo::root\-{now:%Y\-%m\-%d} / \-\-one\-file\-system
 
 # Backup a remote host locally ("pull" style) using sshfs
@@ -212,6 +229,11 @@ $ borg create /path/to/repo::{hostname}\-{user}\-{now} ~
 $ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S} ~
 # As above, but add nanoseconds
 $ borg create /path/to/repo::{hostname}\-{user}\-{now:%Y\-%m\-%dT%H:%M:%S.%f} ~
+
+# Backing up relative paths by moving into the correct directory first
+$ cd /home/user/Documents
+# The root directory of the archive will be "projectA"
+$ borg create /path/to/repo::daily\-projectA\-{now:%Y\-%m\-%d} projectA
 .ft P
 .fi
 .UNINDENT
@@ -222,6 +244,15 @@ The \-\-exclude patterns are not like tar. In tar \-\-exclude .bundler/gems will
 exclude foo/.bundler/gems. In borg it will not, you need to use \-\-exclude
 \(aq*/.bundler/gems\(aq to get the same effect. See \fBborg help patterns\fP for
 more information.
+.sp
+In addition to using \fB\-\-exclude\fP patterns, it is possible to use
+\fB\-\-exclude\-if\-present\fP to specify the name of a filesystem object (e.g. a file
+or folder name) which, when contained within another folder, will prevent the
+containing folder from being backed up.  By default, the containing folder and
+all of its contents will be omitted from the backup.  If, however, you wish to
+only include the objects specified by \fB\-\-exclude\-if\-present\fP in your backup,
+and not include any other contents of the containing folder, this can be enabled
+through using the \fB\-\-keep\-exclude\-tags\fP option.
 .SS Item flags
 .sp
 \fB\-\-list\fP outputs a list of all files, directories and other

docs/man/borg-delete.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-DELETE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-DELETE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-delete \- Delete an existing repository or archives
 .
@@ -60,7 +60,7 @@ print statistics for the deleted archive
 delete only the local cache for the given repository
 .TP
 .B \-\-force
-force deletion of corrupted archives
+force deletion of corrupted archives, use \-\-force \-\-force in case \-\-force does not work.
 .TP
 .B \-\-save\-space
 work slower, but using less space

docs/man/borg-diff.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-DIFF 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-DIFF 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-diff \- Diff contents of two archives
 .
@@ -69,12 +69,6 @@ paths of items inside the archives to compare; patterns are supported
 .SS optional arguments
 .INDENT 0.0
 .TP
-.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
-exclude paths matching PATTERN
-.TP
-.BI \-\-exclude\-from \ EXCLUDEFILE
-read exclude patterns from EXCLUDEFILE, one per line
-.TP
 .B \-\-numeric\-owner
 only consider numeric user and group identifiers
 .TP
@@ -84,6 +78,30 @@ Override check of chunker parameters.
 .B \-\-sort
 Sort the output lines by file path.
 .UNINDENT
+.SS Exclusion options
+.INDENT 0.0
+.TP
+.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
+exclude paths matching PATTERN
+.TP
+.BI \-\-exclude\-from \ EXCLUDEFILE
+read exclude patterns from EXCLUDEFILE, one per line
+.TP
+.B \-\-exclude\-caches
+exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
+.TP
+.BI \-\-exclude\-if\-present \ NAME
+exclude directories that are tagged by containing a filesystem object with the given NAME
+.TP
+.B \-\-keep\-exclude\-tags\fP,\fB  \-\-keep\-tag\-files
+if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
+.TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
+.UNINDENT
 .SH EXAMPLES
 .INDENT 0.0
 .INDENT 3.5

docs/man/borg-extract.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-EXTRACT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-EXTRACT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-extract \- Extract archive contents
 .
@@ -75,6 +75,12 @@ exclude paths matching PATTERN
 .BI \-\-exclude\-from \ EXCLUDEFILE
 read exclude patterns from EXCLUDEFILE, one per line
 .TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
+.TP
 .B \-\-numeric\-owner
 only obey numeric user and group identifiers
 .TP

docs/man/borg-info.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-INFO 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-INFO 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-info \- Show archive details such as disk space used
 .
@@ -57,6 +57,12 @@ See \fIborg\-common(1)\fP for common options of Borg commands.
 .B REPOSITORY_OR_ARCHIVE
 archive or repository to display information about
 .UNINDENT
+.SS optional arguments
+.INDENT 0.0
+.TP
+.B \-\-json
+format output as JSON
+.UNINDENT
 .SS filters
 .INDENT 0.0
 .TP

docs/man/borg-init.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-INIT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-INIT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-init \- Initialize an empty repository
 .

docs/man/borg-key-change-passphrase.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-key-change-passphrase \- Change repository key file passphrase
 .

docs/man/borg-key-export.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-EXPORT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-KEY-EXPORT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-key-export \- Export the repository key for backup
 .
@@ -68,6 +68,9 @@ where to store the backup
 .TP
 .B \-\-paper
 Create an export suitable for printing and later type\-in
+.TP
+.B \-\-qr\-html
+Create an html file suitable for printing and later type\-in or qr scan
 .UNINDENT
 .SH SEE ALSO
 .sp

docs/man/borg-key-import.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-IMPORT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-KEY-IMPORT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-key-import \- Import the repository key from backup
 .

docs/man/borg-key-migrate-to-repokey.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-key-migrate-to-repokey \- Migrate passphrase -> repokey
 .

docs/man/borg-key.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-KEY 1 "2017-02-12" "" "borg backup tool"
+.TH BORG-KEY 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-key \- Manage a keyfile or repokey of a repository
 .

docs/man/borg-list.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-LIST 1 "2017-02-12" "" "borg backup tool"
+.TH BORG-LIST 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-list \- List archive or repository contents
 .
@@ -60,11 +60,8 @@ only print file/directory names, nothing else
 specify format for file listing
 (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")
 .TP
-.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
-exclude paths matching PATTERN
-.TP
-.BI \-\-exclude\-from \ EXCLUDEFILE
-read exclude patterns from EXCLUDEFILE, one per line
+.B \-\-json
+format output as JSON. The form of \-\-format is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available.
 .UNINDENT
 .SS filters
 .INDENT 0.0
@@ -81,6 +78,30 @@ consider first N archives after other filters were applied
 .BI \-\-last \ N
 consider last N archives after other filters were applied
 .UNINDENT
+.SS Exclusion options
+.INDENT 0.0
+.TP
+.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN
+exclude paths matching PATTERN
+.TP
+.BI \-\-exclude\-from \ EXCLUDEFILE
+read exclude patterns from EXCLUDEFILE, one per line
+.TP
+.B \-\-exclude\-caches
+exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
+.TP
+.BI \-\-exclude\-if\-present \ NAME
+exclude directories that are tagged by containing a filesystem object with the given NAME
+.TP
+.B \-\-keep\-exclude\-tags\fP,\fB  \-\-keep\-tag\-files
+if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
+.TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
+.UNINDENT
 .SH EXAMPLES
 .INDENT 0.0
 .INDENT 3.5
@@ -141,7 +162,7 @@ Keys for listing repository archives:
 .INDENT 3.5
 .INDENT 0.0
 .IP \(bu 2
-archive: archive name interpreted as text (might be missing non\-text characters, see barchive)
+archive, name: archive name interpreted as text (might be missing non\-text characters, see barchive)
 .IP \(bu 2
 barchive: verbatim archive name, can contain any character except NUL
 .IP \(bu 2
@@ -183,6 +204,10 @@ size
 .IP \(bu 2
 csize: compressed size
 .IP \(bu 2
+dsize: deduplicated size
+.IP \(bu 2
+dcsize: deduplicated compressed size
+.IP \(bu 2
 num_chunks: number of chunks in this file
 .IP \(bu 2
 unique_chunks: number of unique chunks in this file
@@ -199,10 +224,6 @@ isoctime
 .IP \(bu 2
 isoatime
 .IP \(bu 2
-blake2b
-.IP \(bu 2
-blake2s
-.IP \(bu 2
 md5
 .IP \(bu 2
 sha1
@@ -213,20 +234,8 @@ sha256
 .IP \(bu 2
 sha384
 .IP \(bu 2
-sha3_224
-.IP \(bu 2
-sha3_256
-.IP \(bu 2
-sha3_384
-.IP \(bu 2
-sha3_512
-.IP \(bu 2
 sha512
 .IP \(bu 2
-shake_128
-.IP \(bu 2
-shake_256
-.IP \(bu 2
 archiveid
 .IP \(bu 2
 archivename

docs/man/borg-mount.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-MOUNT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-MOUNT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-mount \- Mount archive or an entire repository as a FUSE filesystem
 .

docs/man/borg-patterns.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PATTERNS 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-PATTERNS 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-patterns \- Details regarding patterns
 .
@@ -32,8 +32,9 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 ..
 .SH DESCRIPTION
 .sp
-Exclusion patterns support four separate styles, fnmatch, shell, regular
-expressions and path prefixes. By default, fnmatch is used. If followed
+File patterns support these styles: fnmatch, shell, regular expressions,
+path prefixes and path full\-matches. By default, fnmatch is used for
+\fI\-\-exclude\fP patterns and shell\-style is used for \fI\-\-pattern\fP\&. If followed
 by a colon (\(aq:\(aq) the first two characters of a pattern are used as a
 style selector. Explicit style selection is necessary when a
 non\-default style is desired or when the desired pattern starts with
@@ -42,12 +43,12 @@ two alphanumeric characters followed by a colon (i.e. \fIaa:something/*\fP).
 \fI\%Fnmatch\fP, selector \fIfm:\fP
 .INDENT 0.0
 .INDENT 3.5
-This is the default style.  These patterns use a variant of shell
-pattern syntax, with \(aq*\(aq matching any number of characters, \(aq?\(aq
-matching any single character, \(aq[...]\(aq matching any single
-character specified, including ranges, and \(aq[!...]\(aq matching any
-character not specified. For the purpose of these patterns, the
-path separator (\(aq\(aq for Windows and \(aq/\(aq on other systems) is not
+This is the default style for \-\-exclude and \-\-exclude\-from.
+These patterns use a variant of shell pattern syntax, with \(aq*\(aq matching
+any number of characters, \(aq?\(aq matching any single character, \(aq[...]\(aq
+matching any single character specified, including ranges, and \(aq[!...]\(aq
+matching any character not specified. For the purpose of these patterns,
+the path separator (\(aq\(aq for Windows and \(aq/\(aq on other systems) is not
 treated specially. Wrap meta\-characters in brackets for a literal
 match (i.e. \fI[?]\fP to match the literal character \fI?\fP). For a path
 to match a pattern, it must completely match from start to end, or
@@ -61,6 +62,7 @@ separator, a \(aq*\(aq is appended before matching is attempted.
 Shell\-style patterns, selector \fIsh:\fP
 .INDENT 0.0
 .INDENT 3.5
+This is the default style for \-\-pattern and \-\-patterns\-from.
 Like fnmatch patterns these are similar to shell patterns. The difference
 is that the pattern may include \fI**/\fP for matching zero or more directory
 levels, \fI*\fP for matching zero or more arbitrary characters with the
@@ -82,7 +84,7 @@ the re module\fP\&.
 .UNINDENT
 .UNINDENT
 .sp
-Prefix path, selector \fIpp:\fP
+Path prefix, selector \fIpp:\fP
 .INDENT 0.0
 .INDENT 3.5
 This pattern style is useful to match whole sub\-directories. The pattern
@@ -90,6 +92,25 @@ This pattern style is useful to match whole sub\-directories. The pattern
 .UNINDENT
 .UNINDENT
 .sp
+Path full\-match, selector \fIpf:\fP
+.INDENT 0.0
+.INDENT 3.5
+This pattern style is useful to match whole paths.
+This is kind of a pseudo pattern as it can not have any variable or
+unspecified parts \- the full, precise path must be given.
+\fIpf:/data/foo.txt\fP matches \fI/data/foo.txt\fP only.
+.sp
+Implementation note: this is implemented via very time\-efficient O(1)
+hashtable lookups (this means you can have huge amounts of such patterns
+without impacting performance much).
+Due to that, this kind of pattern does not respect any context or order.
+If you use such a pattern to include a file, it will always be included
+(if the directory recursion encounters it).
+Other include/exclude patterns that would normally match will be ignored.
+Same logic applies for exclude.
+.UNINDENT
+.UNINDENT
+.sp
 Exclusions can be passed via the command line option \fI\-\-exclude\fP\&. When used
 from within a shell the patterns should be quoted to protect them from
 expansion.
@@ -138,6 +159,47 @@ $ borg create \-\-exclude\-from exclude.txt backup /
 .fi
 .UNINDENT
 .UNINDENT
+.sp
+A more general and easier to use way to define filename matching patterns exists
+with the \fI\-\-pattern\fP and \fI\-\-patterns\-from\fP options. Using these, you may specify
+the backup roots (starting points) and patterns for inclusion/exclusion. A
+root path starts with the prefix \fIR\fP, followed by a path (a plain path, not a
+file pattern). An include rule starts with the prefix +, an exclude rule starts
+with the prefix \-, both followed by a pattern.
+Inclusion patterns are useful to include pathes that are contained in an excluded
+path. The first matching pattern is used so if an include pattern matches before
+an exclude pattern, the file is backed up.
+.sp
+Note that the default pattern style for \fI\-\-pattern\fP and \fI\-\-patterns\-from\fP is
+shell style (\fIsh:\fP), so those patterns behave similar to rsync include/exclude
+patterns. The pattern style can be set via the \fIP\fP prefix.
+.sp
+Patterns (\fI\-\-pattern\fP) and excludes (\fI\-\-exclude\fP) from the command line are
+considered first (in the order of appearance). Then patterns from \fI\-\-patterns\-from\fP
+are added. Exclusion patterns from \fI\-\-exclude\-from\fP files are appended last.
+.sp
+An example \fI\-\-patterns\-from\fP file could look like that:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.nf
+.ft C
+# "sh:" pattern style is the default, so the following line is not needed:
+P sh
+R /
+# can be rebuild
+\- /home/*/.cache
+# they\(aqre downloads for a reason
+\- /home/*/Downloads
+# susan is a nice person
+# include susans home
++ /home/susan
+# don\(aqt backup the other home directories
+\- /home/*
+.ft P
+.fi
+.UNINDENT
+.UNINDENT
 .SH AUTHOR
 The Borg Collective
 .\" Generated by docutils manpage writer.

docs/man/borg-placeholders.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PLACEHOLDERS 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-PLACEHOLDERS 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-placeholders \- Details regarding placeholders
 .

docs/man/borg-prune.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-PRUNE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-PRUNE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-prune \- Prune repository archives according to specified rules
 .

docs/man/borg-recreate.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-RECREATE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-RECREATE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-recreate \- Re-create archives
 .
@@ -39,9 +39,9 @@ Recreate the contents of existing archives.
 .sp
 This is an \fIexperimental\fP feature. Do \fInot\fP use this on your only backup.
 .sp
-\-\-exclude, \-\-exclude\-from and PATH have the exact same semantics
-as in "borg create". If PATHs are specified the resulting archive
-will only contain files from these PATHs.
+\-\-exclude, \-\-exclude\-from, \-\-exclude\-if\-present, \-\-keep\-exclude\-tags, and PATH
+have the exact same semantics as in "borg create". If PATHs are specified the
+resulting archive will only contain files from these PATHs.
 .sp
 Note that all paths in an archive are relative, therefore absolute patterns/paths
 will \fInot\fP match (\-\-exclude, \-\-exclude\-from, \-\-compression\-from, PATHs).
@@ -114,10 +114,16 @@ read exclude patterns from EXCLUDEFILE, one per line
 exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP)
 .TP
 .BI \-\-exclude\-if\-present \ NAME
-exclude directories that are tagged by containing a filesystem object with                                          the given NAME
+exclude directories that are tagged by containing a filesystem object with the given NAME
 .TP
 .B \-\-keep\-exclude\-tags\fP,\fB  \-\-keep\-tag\-files
-keep tag objects (i.e.: arguments to \-\-exclude\-if\-present) in otherwise                                          excluded caches/directories
+if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive
+.TP
+.BI \-\-pattern \ PATTERN
+include/exclude paths matching PATTERN
+.TP
+.BI \-\-patterns\-from \ PATTERNFILE
+read include/exclude patterns from PATTERNFILE, one per line
 .UNINDENT
 .SS Archive options
 .INDENT 0.0

docs/man/borg-rename.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-RENAME 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-RENAME 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-rename \- Rename an existing archive
 .

docs/man/borg-serve.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-SERVE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-SERVE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-serve \- Start in server mode. This command is usually not used manually.
 .

docs/man/borg-umount.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-UMOUNT 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-UMOUNT 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-umount \- un-mount the FUSE filesystem
 .

docs/man/borg-upgrade.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-UPGRADE 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-UPGRADE 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-upgrade \- upgrade a repository from a previous version
 .

docs/man/borg-with-lock.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH BORG-WITH-LOCK 1 "2017-02-11" "" "borg backup tool"
+.TH BORG-WITH-LOCK 1 "2017-03-26" "" "borg backup tool"
 .SH NAME
 borg-with-lock \- run a user specified command with the repository lock held
 .

docs/man/borg.1

@@ -336,6 +336,10 @@ Main usecase for this is to fully automate \fBborg change\-passphrase\fP\&.
 .B BORG_DISPLAY_PASSPHRASE
 When set, use the value to answer the "display the passphrase for verification" question when defining a new passphrase for encrypted repositories.
 .TP
+.B BORG_HOSTNAME_IS_UNIQUE=yes
+Use this to assert that your hostname is unique.
+Borg will then automatically remove locks that it could determine to be stale.
+.TP
 .B BORG_LOGGING_CONF
 When set, use the given filename as \fI\%INI\fP\-style logging configuration.
 .TP
@@ -344,7 +348,7 @@ When set, use this command instead of \fBssh\fP\&. This can be used to specify s
 a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options.
 .TP
 .B BORG_REMOTE_PATH
-When set, use the given path/filename as remote path (default is "borg").
+When set, use the given path as borg executable on the remote (defaults to "borg" if unset).
 Using \fB\-\-remote\-path PATH\fP commandline option overrides the environment variable.
 .TP
 .B BORG_FILES_CACHE_TTL

docs/usage/break-lock.rst.inc

@@ -20,4 +20,4 @@ Description
 
 This command breaks the repository and cache locks.
 Please use carefully and only while no borg process (on any machine) is
-trying to access the Cache or the Repository.
+trying to access the Cache or the Repository.

docs/usage/change-passphrase.rst.inc

@@ -19,4 +19,4 @@ Description
 ~~~~~~~~~~~
 
 The key files used for repository encryption are optionally passphrase
-protected. This command can be used to change this passphrase.
+protected. This command can be used to change this passphrase.

docs/usage/check.rst.inc

@@ -86,4 +86,4 @@ repository, decrypting and decompressing it. This is a cryptographic verificatio
 which will detect (accidental) corruption. For encrypted repositories it is
 tamper-resistant as well, unless the attacker has access to the keys.
 
-It is also very slow.
+It is also very slow.

docs/usage/common-options.rst.inc

@@ -12,6 +12,8 @@
         | enable debug output, work on log level DEBUG
     ``--debug-topic TOPIC``
         | enable TOPIC debugging (can be specified multiple times). The logger path is borg.debug.<TOPIC> if TOPIC is not fully qualified.
+    ``--log-json``
+        | Output one JSON object per log line instead of formatted text.
     ``--lock-wait N``
         | wait for the lock, but max. N seconds (default: 1).
     ``--show-version``
@@ -23,7 +25,7 @@
     ``--umask M``
         | set umask to M (local and remote, default: 0077)
     ``--remote-path PATH``
-        | set remote path to executable (default: "borg")
+        | use PATH as borg executable on the remote (default: "borg")
     ``--remote-ratelimit rate``
         | set remote network upload rate limit in kiByte/s (default: 0=unlimited)
     ``--consider-part-files``

docs/usage/create.rst.inc

@@ -25,6 +25,8 @@ optional arguments
         | output verbose list of items (files, dirs, ...)
     ``--filter STATUSCHARS``
         | only display items with the given status characters
+    ``--json``
+        | output stats as JSON (implies --stats)
 
 `Common options`_
     |
@@ -39,11 +41,15 @@ Exclusion options
     ``--exclude-if-present NAME``
         | exclude directories that are tagged by containing a filesystem object with the given NAME
     ``--keep-exclude-tags``, ``--keep-tag-files``
-        | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise excluded caches/directories
+        | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive
+    ``--pattern PATTERN``
+        | include/exclude paths matching PATTERN
+    ``--patterns-from PATTERNFILE``
+        | read include/exclude patterns from PATTERNFILE, one per line
 
 Filesystem options
     ``-x``, ``--one-file-system``
-        | stay in same file system, do not cross mount points
+        | stay in the same file system and do not store mount points of other file systems
     ``--numeric-owner``
         | only store numeric user and group identifiers
     ``--noatime``
@@ -73,9 +79,12 @@ Description
 ~~~~~~~~~~~
 
 This command creates a backup archive containing all files found while recursively
-traversing all paths specified. When giving '-' as path, borg will read data
-from standard input and create a file 'stdin' in the created archive from that
-data.
+traversing all paths specified. Paths are added to the archive as they are given,
+that means if relative paths are desired, the command has to be run from the correct
+directory.
+
+When giving '-' as path, borg will read data from standard input and create a
+file 'stdin' in the created archive from that data.
 
 The archive will consume almost no disk space for files or parts of files that
 have already been stored in other archives.
@@ -92,6 +101,11 @@ not provide correct inode information the --ignore-inode flag can be used. This
 potentially decreases reliability of change detection, while avoiding always reading
 all files on these file systems.
 
+The mount points of filesystems or filesystem snapshots should be the same for every
+creation of a new archive to ensure fast operation. This is because the file cache that
+is used to determine changed files quickly uses absolute filenames.
+If this is not possible, consider creating a bind mount to a stable location.
+
 See the output of the "borg help patterns" command for more help on exclude patterns.
 See the output of the "borg help placeholders" command for more help on placeholders.
 
@@ -102,6 +116,15 @@ exclude foo/.bundler/gems. In borg it will not, you need to use --exclude
 '\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for
 more information.
 
+In addition to using ``--exclude`` patterns, it is possible to use
+``--exclude-if-present`` to specify the name of a filesystem object (e.g. a file
+or folder name) which, when contained within another folder, will prevent the
+containing folder from being backed up.  By default, the containing folder and
+all of its contents will be omitted from the backup.  If, however, you wish to
+only include the objects specified by ``--exclude-if-present`` in your backup,
+and not include any other contents of the containing folder, this can be enabled
+through using the ``--keep-exclude-tags`` option.
+
 Item flags
 ++++++++++
 

docs/usage/delete.rst.inc

@@ -20,7 +20,7 @@ optional arguments
     ``-c``, ``--cache-only``
         | delete only the local cache for the given repository
     ``--force``
-        | force deletion of corrupted archives
+        | force deletion of corrupted archives, use --force --force in case --force does not work.
     ``--save-space``
         | work slower, but using less space
 
@@ -42,4 +42,4 @@ Description
 
 This command deletes an archive from the repository or the complete repository.
 Disk space is reclaimed accordingly. If you delete the complete repository, the
-local cache for it (if any) is also deleted.
+local cache for it (if any) is also deleted.

docs/usage/diff.rst.inc

@@ -17,10 +17,6 @@ positional arguments
         paths of items inside the archives to compare; patterns are supported
 
 optional arguments
-    ``-e PATTERN``, ``--exclude PATTERN``
-        | exclude paths matching PATTERN
-    ``--exclude-from EXCLUDEFILE``
-        | read exclude patterns from EXCLUDEFILE, one per line
     ``--numeric-owner``
         | only consider numeric user and group identifiers
     ``--same-chunker-params``
@@ -31,6 +27,22 @@ optional arguments
 `Common options`_
     |
 
+Exclusion options
+    ``-e PATTERN``, ``--exclude PATTERN``
+        | exclude paths matching PATTERN
+    ``--exclude-from EXCLUDEFILE``
+        | read exclude patterns from EXCLUDEFILE, one per line
+    ``--exclude-caches``
+        | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html)
+    ``--exclude-if-present NAME``
+        | exclude directories that are tagged by containing a filesystem object with the given NAME
+    ``--keep-exclude-tags``, ``--keep-tag-files``
+        | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive
+    ``--pattern PATTERN``
+        | include/exclude paths matching PATTERN
+    ``--patterns-from PATTERNFILE``
+        | read include/exclude patterns from PATTERNFILE, one per line
+
 Description
 ~~~~~~~~~~~
 
@@ -49,4 +61,4 @@ If you did not create the archives with different chunker params,
 pass --same-chunker-params.
 Note that the chunker params changed from Borg 0.xx to 1.0.
 
-See the output of the "borg help patterns" command for more help on exclude patterns.
+See the output of the "borg help patterns" command for more help on exclude patterns.

docs/usage/extract.rst.inc

@@ -25,6 +25,10 @@ optional arguments
         | exclude paths matching PATTERN
     ``--exclude-from EXCLUDEFILE``
         | read exclude patterns from EXCLUDEFILE, one per line
+    ``--pattern PATTERN``
+        | include/exclude paths matching PATTERN
+    ``--patterns-from PATTERNFILE``
+        | read include/exclude patterns from PATTERNFILE, one per line
     ``--numeric-owner``
         | only obey numeric user and group identifiers
     ``--strip-components NUMBER``
@@ -49,4 +53,4 @@ See the output of the "borg help patterns" command for more help on exclude patt
 
 By using ``--dry-run``, you can do all extraction steps except actually writing the
 output data: reading metadata and data chunks from the repo, checking the hash/hmac,
-decrypting, decompressing.
+decrypting, decompressing.

docs/usage/help.rst.inc

@@ -6,8 +6,9 @@ borg help patterns
 ~~~~~~~~~~~~~~~~~~
 
 
-Exclusion patterns support four separate styles, fnmatch, shell, regular
-expressions and path prefixes. By default, fnmatch is used. If followed
+File patterns support these styles: fnmatch, shell, regular expressions,
+path prefixes and path full-matches. By default, fnmatch is used for
+`--exclude` patterns and shell-style is used for `--pattern`. If followed
 by a colon (':') the first two characters of a pattern are used as a
 style selector. Explicit style selection is necessary when a
 non-default style is desired or when the desired pattern starts with
@@ -15,12 +16,12 @@ two alphanumeric characters followed by a colon (i.e. `aa:something/*`).
 
 `Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector `fm:`
 
-    This is the default style.  These patterns use a variant of shell
-    pattern syntax, with '*' matching any number of characters, '?'
-    matching any single character, '[...]' matching any single
-    character specified, including ranges, and '[!...]' matching any
-    character not specified. For the purpose of these patterns, the
-    path separator ('\' for Windows and '/' on other systems) is not
+    This is the default style for --exclude and --exclude-from.
+    These patterns use a variant of shell pattern syntax, with '*' matching
+    any number of characters, '?' matching any single character, '[...]'
+    matching any single character specified, including ranges, and '[!...]'
+    matching any character not specified. For the purpose of these patterns,
+    the path separator ('\' for Windows and '/' on other systems) is not
     treated specially. Wrap meta-characters in brackets for a literal
     match (i.e. `[?]` to match the literal character `?`). For a path
     to match a pattern, it must completely match from start to end, or
@@ -31,6 +32,7 @@ two alphanumeric characters followed by a colon (i.e. `aa:something/*`).
 
 Shell-style patterns, selector `sh:`
 
+    This is the default style for --pattern and --patterns-from.
     Like fnmatch patterns these are similar to shell patterns. The difference
     is that the pattern may include `**/` for matching zero or more directory
     levels, `*` for matching zero or more arbitrary characters with the
@@ -47,11 +49,27 @@ Regular expressions, selector `re:`
     regular expression syntax is described in the `Python documentation for
     the re module <https://docs.python.org/3/library/re.html>`_.
 
-Prefix path, selector `pp:`
+Path prefix, selector `pp:`
 
     This pattern style is useful to match whole sub-directories. The pattern
     `pp:/data/bar` matches `/data/bar` and everything therein.
 
+Path full-match, selector `pf:`
+
+    This pattern style is useful to match whole paths.
+    This is kind of a pseudo pattern as it can not have any variable or
+    unspecified parts - the full, precise path must be given.
+    `pf:/data/foo.txt` matches `/data/foo.txt` only.
+
+    Implementation note: this is implemented via very time-efficient O(1)
+    hashtable lookups (this means you can have huge amounts of such patterns
+    without impacting performance much).
+    Due to that, this kind of pattern does not respect any context or order.
+    If you use such a pattern to include a file, it will always be included
+    (if the directory recursion encounters it).
+    Other include/exclude patterns that would normally match will be ignored.
+    Same logic applies for exclude.
+
 Exclusions can be passed via the command line option `--exclude`. When used
 from within a shell the patterns should be quoted to protect them from
 expansion.
@@ -93,6 +111,40 @@ Examples::
     EOF
     $ borg create --exclude-from exclude.txt backup /
 
+
+A more general and easier to use way to define filename matching patterns exists
+with the `--pattern` and `--patterns-from` options. Using these, you may specify
+the backup roots (starting points) and patterns for inclusion/exclusion. A
+root path starts with the prefix `R`, followed by a path (a plain path, not a
+file pattern). An include rule starts with the prefix +, an exclude rule starts
+with the prefix -, both followed by a pattern.
+Inclusion patterns are useful to include pathes that are contained in an excluded
+path. The first matching pattern is used so if an include pattern matches before
+an exclude pattern, the file is backed up.
+
+Note that the default pattern style for `--pattern` and `--patterns-from` is
+shell style (`sh:`), so those patterns behave similar to rsync include/exclude
+patterns. The pattern style can be set via the `P` prefix.
+
+Patterns (`--pattern`) and excludes (`--exclude`) from the command line are
+considered first (in the order of appearance). Then patterns from `--patterns-from`
+are added. Exclusion patterns from `--exclude-from` files are appended last.
+
+An example `--patterns-from` file could look like that::
+
+    # "sh:" pattern style is the default, so the following line is not needed:
+    P sh
+    R /
+    # can be rebuild
+    - /home/*/.cache
+    # they're downloads for a reason
+    - /home/*/Downloads
+    # susan is a nice person
+    # include susans home
+    + /home/susan
+    # don't backup the other home directories
+    - /home/*
+
 .. _borg_placeholders:
 
 borg help placeholders
@@ -156,17 +208,17 @@ borg help compression
 ~~~~~~~~~~~~~~~~~~~~~
 
 
-Compression is off by default, if you want some, you have to specify what you want.
+Compression is lz4 by default. If you want something else, you have to specify what you want.
 
 Valid compression specifiers are:
 
 none
 
-    Do not compress. (default)
+    Do not compress.
 
 lz4
 
-    Use lz4 compression. High speed, low compression.
+    Use lz4 compression. High speed, low compression. (default)
 
 zlib[,L]
 

docs/usage/info.rst.inc

@@ -12,6 +12,10 @@ positional arguments
     REPOSITORY_OR_ARCHIVE
         archive or repository to display information about
 
+optional arguments
+    ``--json``
+        | format output as JSON
+
 `Common options`_
     |
 
@@ -37,4 +41,4 @@ are meaning different things:
 This archive / deduplicated size = amount of data stored ONLY for this archive
                                  = unique chunks of this archive.
 All archives / deduplicated size = amount of data stored in the repo
-                                 = all chunks in the repository.
+                                 = all chunks in the repository.

docs/usage/init.rst.inc

@@ -97,4 +97,4 @@ hardware-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs,
 which makes `authenticated` faster than `none`.
 
 On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster
-than BLAKE2b-256 there.
+than BLAKE2b-256 there.

docs/usage/key_change-passphrase.rst.inc

@@ -19,4 +19,4 @@ Description
 ~~~~~~~~~~~
 
 The key files used for repository encryption are optionally passphrase
-protected. This command can be used to change this passphrase.
+protected. This command can be used to change this passphrase.

docs/usage/key_export.rst.inc

@@ -17,6 +17,8 @@ positional arguments
 optional arguments
     ``--paper``
         | Create an export suitable for printing and later type-in
+    ``--qr-html``
+        | Create an html file suitable for printing and later type-in or qr scan
 
 `Common options`_
     |
@@ -40,4 +42,4 @@ data backup.
 For repositories using the repokey encryption the key is saved in the
 repository in the config file. A backup is thus not strictly needed,
 but guards against the repository becoming inaccessible if the file
-is damaged for some reason.
+is damaged for some reason.

docs/usage/key_import.rst.inc

@@ -29,4 +29,4 @@ export command.
 
 If the ``--paper`` option is given, the import will be an interactive
 process in which each line is checked for plausibility before
-proceeding to the next line. For this format PATH must not be given.
+proceeding to the next line. For this format PATH must not be given.

docs/usage/key_migrate-to-repokey.rst.inc

@@ -33,4 +33,4 @@ use, but you may also use a different one.
 
 After migrating to repokey mode, you can change the passphrase at any time.
 But please note: the secrets will always stay the same and they could always
-be derived from your (old) passphrase-mode passphrase.
+be derived from your (old) passphrase-mode passphrase.

docs/usage/list.rst.inc

@@ -20,10 +20,8 @@ optional arguments
     ``--format``, ``--list-format``
         | specify format for file listing
         |                                 (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")
-    ``-e PATTERN``, ``--exclude PATTERN``
-        | exclude paths matching PATTERN
-    ``--exclude-from EXCLUDEFILE``
-        | read exclude patterns from EXCLUDEFILE, one per line
+    ``--json``
+        | format output as JSON. The form of --format is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available.
 
 `Common options`_
     |
@@ -38,6 +36,22 @@ filters
     ``--last N``
         | consider last N archives after other filters were applied
 
+Exclusion options
+    ``-e PATTERN``, ``--exclude PATTERN``
+        | exclude paths matching PATTERN
+    ``--exclude-from EXCLUDEFILE``
+        | read exclude patterns from EXCLUDEFILE, one per line
+    ``--exclude-caches``
+        | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html)
+    ``--exclude-if-present NAME``
+        | exclude directories that are tagged by containing a filesystem object with the given NAME
+    ``--keep-exclude-tags``, ``--keep-tag-files``
+        | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive
+    ``--pattern PATTERN``
+        | include/exclude paths matching PATTERN
+    ``--patterns-from PATTERNFILE``
+        | read include/exclude patterns from PATTERNFILE, one per line
+
 Description
 ~~~~~~~~~~~
 
@@ -45,6 +59,8 @@ This command lists the contents of a repository or an archive.
 
 See the "borg help patterns" command for more help on exclude patterns.
 
+.. man NOTES
+
 The following keys are available for --format:
 
  - NEWLINE: OS dependent line separator
@@ -57,7 +73,7 @@ The following keys are available for --format:
 
 Keys for listing repository archives:
 
- - archive: archive name interpreted as text (might be missing non-text characters, see barchive)
+ - archive, name: archive name interpreted as text (might be missing non-text characters, see barchive)
  - barchive: verbatim archive name, can contain any character except NUL
  - time: time of creation of the archive
  - id: internal ID of the archive
@@ -78,6 +94,8 @@ Keys for listing archive files:
 
  - size
  - csize: compressed size
+ - dsize: deduplicated size
+ - dcsize: deduplicated compressed size
  - num_chunks: number of chunks in this file
  - unique_chunks: number of unique chunks in this file
 

docs/usage/mount.rst.inc

@@ -67,4 +67,4 @@ Unmounting in these cases could cause an active rsync or similar process
 to unintentionally delete data.
 
 When running in the foreground ^C/SIGINT unmounts cleanly, but other
-signals or crashes do not.
+signals or crashes do not.

docs/usage/prune.rst.inc

@@ -87,4 +87,4 @@ negative number of archives to keep means that there is no limit.
 
 The "--keep-last N" option is doing the same as "--keep-secondly N" (and it will
 keep the last N archives under the assumption that you do not create more than one
-backup archive in the same second).
+backup archive in the same second).

docs/usage/recreate.rst.inc

@@ -37,9 +37,13 @@ Exclusion options
     ``--exclude-caches``
         | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html)
     ``--exclude-if-present NAME``
-        | exclude directories that are tagged by containing a filesystem object with                                          the given NAME
+        | exclude directories that are tagged by containing a filesystem object with the given NAME
     ``--keep-exclude-tags``, ``--keep-tag-files``
-        | keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise                                          excluded caches/directories
+        | if tag objects are specified with --exclude-if-present, don't omit the tag objects themselves from the backup archive
+    ``--pattern PATTERN``
+        | include/exclude paths matching PATTERN
+    ``--patterns-from PATTERNFILE``
+        | read include/exclude patterns from PATTERNFILE, one per line
 
 Archive options
     ``--target TARGET``
@@ -66,9 +70,9 @@ Recreate the contents of existing archives.
 
 This is an *experimental* feature. Do *not* use this on your only backup.
 
---exclude, --exclude-from and PATH have the exact same semantics
-as in "borg create". If PATHs are specified the resulting archive
-will only contain files from these PATHs.
+--exclude, --exclude-from, --exclude-if-present, --keep-exclude-tags, and PATH
+have the exact same semantics as in "borg create". If PATHs are specified the
+resulting archive will only contain files from these PATHs.
 
 Note that all paths in an archive are relative, therefore absolute patterns/paths
 will *not* match (--exclude, --exclude-from, --compression-from, PATHs).
@@ -97,4 +101,4 @@ With --target the original archive is not replaced, instead a new archive is cre
 When rechunking space usage can be substantial, expect at least the entire
 deduplicated size of the archives using the previous chunker params.
 When recompressing expect approx. (throughput / checkpoint-interval) in space usage,
-assuming all chunks are recompressed.
+assuming all chunks are recompressed.

docs/usage/rename.rst.inc

@@ -22,4 +22,4 @@ Description
 
 This command renames an archive in the repository.
 
-This results in a different archive ID.
+This results in a different archive ID.

docs/usage/serve.rst.inc

@@ -20,4 +20,4 @@ optional arguments
 Description
 ~~~~~~~~~~~
 
-This command starts a repository server process. This command is usually not used manually.
+This command starts a repository server process. This command is usually not used manually.

docs/usage/umount.rst.inc

@@ -21,4 +21,4 @@ Description
 This command un-mounts a FUSE filesystem that was mounted with ``borg mount``.
 
 This is a convenience wrapper that just calls the platform-specific shell
-command - usually this is either umount or fusermount -u.
+command - usually this is either umount or fusermount -u.

docs/usage/with-lock.rst.inc

@@ -31,4 +31,4 @@ code as borg's return code.
 
 Note: if you copy a repository with the lock held, the lock will be present in
       the copy, obviously. Thus, before using borg on the copy, you need to
-      use "borg break-lock" on it.
+      use "borg break-lock" on it.

src/borg/archiver.py

@@ -325,6 +325,7 @@ class Archiver:
         return EXIT_SUCCESS
 
     def do_benchmark_crud(self, args):
+        """Benchmark Create, Read, Update, Delete for archives."""
         def measurement_run(repo, path):
             archive = repo + '::borg-benchmark-crud'
             compression = '--compression=none'
@@ -1561,8 +1562,8 @@ class Archiver:
 
     helptext = collections.OrderedDict()
     helptext['patterns'] = textwrap.dedent('''
-        File patterns support four separate styles: fnmatch, shell, regular
-        expressions and path prefixes. By default, fnmatch is used for
+        File patterns support these styles: fnmatch, shell, regular expressions,
+        path prefixes and path full-matches. By default, fnmatch is used for
         `--exclude` patterns and shell-style is used for `--pattern`. If followed
         by a colon (':') the first two characters of a pattern are used as a
         style selector. Explicit style selection is necessary when a

src/borg/compress.pyx

@@ -104,13 +104,16 @@ class LZ4(CompressorBase):
         # allocate more if isize * 3 is already bigger, to avoid having to resize often.
         osize = max(int(1.1 * 2**23), isize * 3)
         while True:
-            buf = buffer.get(osize)
+            try:
+                buf = buffer.get(osize)
+            except MemoryError:
+                raise DecompressionError('MemoryError')
             dest = <char *> buf
             with nogil:
                 rsize = LZ4_decompress_safe(source, dest, isize, osize)
             if rsize >= 0:
                 break
-            if osize > 2 ** 30:
+            if osize > 2 ** 27:  # 128MiB (should be enough, considering max. repo obj size and very good compression)
                 # this is insane, get out of here
                 raise DecompressionError('lz4 decompress failed')
             # likely the buffer was too small, get a bigger one: