|
@@ -1,8 +1,5 @@
|
|
|
.\" Man page generated from reStructuredText.
|
|
|
.
|
|
|
-.TH BORG-PATTERNS 1 "2022-04-14" "" "borg backup tool"
|
|
|
-.SH NAME
|
|
|
-borg-patterns \- Details regarding patterns
|
|
|
.
|
|
|
.nr rst2man-indent-level 0
|
|
|
.
|
|
@@ -30,45 +27,47 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
|
|
|
.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
|
|
|
.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
|
|
|
..
|
|
|
+.TH "BORG-PATTERNS" 1 "2022-06-25" "" "borg backup tool"
|
|
|
+.SH NAME
|
|
|
+borg-patterns \- Details regarding patterns
|
|
|
.SH DESCRIPTION
|
|
|
.sp
|
|
|
-The path/filenames used as input for the pattern matching start from the
|
|
|
-currently active recursion root. You usually give the recursion root(s)
|
|
|
-when invoking borg and these can be either relative or absolute paths.
|
|
|
-.sp
|
|
|
-So, when you give \fIrelative/\fP as root, the paths going into the matcher
|
|
|
-will look like \fIrelative/.../file.ext\fP\&. When you give \fI/absolute/\fP as
|
|
|
-root, they will look like \fI/absolute/.../file.ext\fP\&.
|
|
|
-.sp
|
|
|
-File paths in Borg archives are always stored normalized and relative.
|
|
|
-This means that e.g. \fBborg create /path/to/repo ../some/path\fP will
|
|
|
-store all files as \fIsome/path/.../file.ext\fP and \fBborg create
|
|
|
-/path/to/repo /home/user\fP will store all files as
|
|
|
-\fIhome/user/.../file.ext\fP\&.
|
|
|
-.sp
|
|
|
-A directory exclusion pattern can end either with or without a slash (\(aq/\(aq).
|
|
|
-If it ends with a slash, such as \fIsome/path/\fP, the directory will be
|
|
|
-included but not its content. If it does not end with a slash, such as
|
|
|
-\fIsome/path\fP, both the directory and content will be excluded.
|
|
|
+When specifying one or more file paths in a Borg command that supports
|
|
|
+patterns for the respective option or argument, you can apply the
|
|
|
+patterns described here to include only desired files and/or exclude
|
|
|
+unwanted ones. Patterns can be used
|
|
|
+.INDENT 0.0
|
|
|
+.IP \(bu 2
|
|
|
+for \fB\-\-exclude\fP option,
|
|
|
+.IP \(bu 2
|
|
|
+in the file given with \fB\-\-exclude\-from\fP option,
|
|
|
+.IP \(bu 2
|
|
|
+for \fB\-\-pattern\fP option,
|
|
|
+.IP \(bu 2
|
|
|
+in the file given with \fB\-\-patterns\-from\fP option and
|
|
|
+.IP \(bu 2
|
|
|
+for \fBPATH\fP arguments that explicitly support them.
|
|
|
+.UNINDENT
|
|
|
.sp
|
|
|
-File patterns support these styles: fnmatch, shell, regular expressions,
|
|
|
-path prefixes and path full\-matches. By default, fnmatch is used for
|
|
|
-\fB\-\-exclude\fP patterns and shell\-style is used for the \fB\-\-pattern\fP
|
|
|
-option. For commands that support patterns in their \fBPATH\fP argument
|
|
|
-like (\fBborg list\fP), the default pattern is path prefix.
|
|
|
+Borg always stores all file paths normalized and relative to the
|
|
|
+current recursion root. The recursion root is also named \fBPATH\fP in
|
|
|
+Borg commands like \fIborg create\fP that do a file discovery, so do not
|
|
|
+confuse the root with the \fBPATH\fP argument of e.g. \fIborg extract\fP\&.
|
|
|
.sp
|
|
|
-Starting with Borg 1.2, for all but regular expression pattern matching
|
|
|
-styles, all paths are treated as relative, meaning that a leading path
|
|
|
-separator is removed after normalizing and before matching. This allows
|
|
|
-you to use absolute or relative patterns arbitrarily.
|
|
|
+Starting with Borg 1.2, paths that are matched against patterns always
|
|
|
+appear relative. If you give \fB/absolute/\fP as root, the paths going
|
|
|
+into the matcher will look relative like \fBabsolute/.../file.ext\fP\&.
|
|
|
+If you give \fB\&../some/path\fP as root, the paths will look like
|
|
|
+\fBsome/path/.../file.ext\fP\&.
|
|
|
.sp
|
|
|
-If followed by a colon (\(aq:\(aq) the first two characters of a pattern are
|
|
|
-used as a style selector. Explicit style selection is necessary when a
|
|
|
-non\-default style is desired or when the desired pattern starts with
|
|
|
-two alphanumeric characters followed by a colon (i.e. \fIaa:something/*\fP).
|
|
|
+File patterns support five different styles. If followed by a colon \(aq:\(aq,
|
|
|
+the first two characters of a pattern are used as a style selector.
|
|
|
+Explicit style selection is necessary if a non\-default style is desired
|
|
|
+or when the desired pattern starts with two alphanumeric characters
|
|
|
+followed by a colon (i.e. \fBaa:something/*\fP).
|
|
|
.INDENT 0.0
|
|
|
.TP
|
|
|
-.B \fI\%Fnmatch\fP, selector \fIfm:\fP
|
|
|
+.B \fI\%Fnmatch\fP, selector \fBfm:\fP
|
|
|
This is the default style for \fB\-\-exclude\fP and \fB\-\-exclude\-from\fP\&.
|
|
|
These patterns use a variant of shell pattern syntax, with \(aq*\(aq matching
|
|
|
any number of characters, \(aq?\(aq matching any single character, \(aq[...]\(aq
|
|
@@ -76,7 +75,7 @@ matching any single character specified, including ranges, and \(aq[!...]\(aq
|
|
|
matching any character not specified. For the purpose of these patterns,
|
|
|
the path separator (backslash for Windows and \(aq/\(aq on other systems) is not
|
|
|
treated specially. Wrap meta\-characters in brackets for a literal
|
|
|
-match (i.e. \fI[?]\fP to match the literal character \fI?\fP). For a path
|
|
|
+match (i.e. \fB[?]\fP to match the literal character \(aq?\(aq). For a path
|
|
|
to match a pattern, the full path must match, or it must match
|
|
|
from the start of the full path to just before a path separator. Except
|
|
|
for the root path, paths will never end in the path separator when
|
|
@@ -84,33 +83,31 @@ matching is attempted. Thus, if a given pattern ends in a path
|
|
|
separator, a \(aq*\(aq is appended before matching is attempted. A leading
|
|
|
path separator is always removed.
|
|
|
.TP
|
|
|
-.B Shell\-style patterns, selector \fIsh:\fP
|
|
|
+.B Shell\-style patterns, selector \fBsh:\fP
|
|
|
This is the default style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP\&.
|
|
|
Like fnmatch patterns these are similar to shell patterns. The difference
|
|
|
-is that the pattern may include \fI**/\fP for matching zero or more directory
|
|
|
-levels, \fI*\fP for matching zero or more arbitrary characters with the
|
|
|
+is that the pattern may include \fB**/\fP for matching zero or more directory
|
|
|
+levels, \fB*\fP for matching zero or more arbitrary characters with the
|
|
|
exception of any path separator. A leading path separator is always removed.
|
|
|
.TP
|
|
|
-.B Regular expressions, selector \fIre:\fP
|
|
|
-Regular expressions similar to those found in Perl are supported. Unlike
|
|
|
-shell patterns regular expressions are not required to match the full
|
|
|
+.B \fI\%Regular expressions\fP, selector \fBre:\fP
|
|
|
+Unlike shell patterns, regular expressions are not required to match the full
|
|
|
path and any substring match is sufficient. It is strongly recommended to
|
|
|
anchor patterns to the start (\(aq^\(aq), to the end (\(aq$\(aq) or both. Path
|
|
|
separators (backslash for Windows and \(aq/\(aq on other systems) in paths are
|
|
|
-always normalized to a forward slash (\(aq/\(aq) before applying a pattern. The
|
|
|
-regular expression syntax is described in the \fI\%Python documentation for
|
|
|
-the re module\fP\&.
|
|
|
+always normalized to a forward slash \(aq/\(aq before applying a pattern.
|
|
|
.TP
|
|
|
-.B Path prefix, selector \fIpp:\fP
|
|
|
+.B Path prefix, selector \fBpp:\fP
|
|
|
This pattern style is useful to match whole sub\-directories. The pattern
|
|
|
-\fIpp:root/somedir\fP matches \fIroot/somedir\fP and everything therein. A leading
|
|
|
-path separator is always removed.
|
|
|
+\fBpp:root/somedir\fP matches \fBroot/somedir\fP and everything therein.
|
|
|
+A leading path separator is always removed.
|
|
|
.TP
|
|
|
-.B Path full\-match, selector \fIpf:\fP
|
|
|
+.B Path full\-match, selector \fBpf:\fP
|
|
|
This pattern style is (only) useful to match full paths.
|
|
|
This is kind of a pseudo pattern as it can not have any variable or
|
|
|
-unspecified parts \- the full path must be given. \fIpf:root/file.ext\fP matches
|
|
|
-\fIroot/file.ext\fP only. A leading path separator is always removed.
|
|
|
+unspecified parts \- the full path must be given. \fBpf:root/file.ext\fP
|
|
|
+matches \fBroot/file.ext\fP only. A leading path separator is always
|
|
|
+removed.
|
|
|
.sp
|
|
|
Implementation note: this is implemented via very time\-efficient O(1)
|
|
|
hashtable lookups (this means you can have huge amounts of such patterns
|
|
@@ -125,12 +122,12 @@ Same logic applies for exclude.
|
|
|
\fBNOTE:\fP
|
|
|
.INDENT 0.0
|
|
|
.INDENT 3.5
|
|
|
-\fIre:\fP, \fIsh:\fP and \fIfm:\fP patterns are all implemented on top of the Python SRE
|
|
|
-engine. It is very easy to formulate patterns for each of these types which
|
|
|
-requires an inordinate amount of time to match paths. If untrusted users
|
|
|
-are able to supply patterns, ensure they cannot supply \fIre:\fP patterns.
|
|
|
-Further, ensure that \fIsh:\fP and \fIfm:\fP patterns only contain a handful of
|
|
|
-wildcards at most.
|
|
|
+\fBre:\fP, \fBsh:\fP and \fBfm:\fP patterns are all implemented on top of
|
|
|
+the Python SRE engine. It is very easy to formulate patterns for each
|
|
|
+of these types which requires an inordinate amount of time to match
|
|
|
+paths. If untrusted users are able to supply patterns, ensure they
|
|
|
+cannot supply \fBre:\fP patterns. Further, ensure that \fBsh:\fP and
|
|
|
+\fBfm:\fP patterns only contain a handful of wildcards at most.
|
|
|
.UNINDENT
|
|
|
.UNINDENT
|
|
|
.sp
|
|
@@ -139,8 +136,8 @@ from within a shell, the patterns should be quoted to protect them from
|
|
|
expansion.
|
|
|
.sp
|
|
|
The \fB\-\-exclude\-from\fP option permits loading exclusion patterns from a text
|
|
|
-file with one pattern per line. Lines empty or starting with the number sign
|
|
|
-(\(aq#\(aq) after removing whitespace on both ends are ignored. The optional style
|
|
|
+file with one pattern per line. Lines empty or starting with the hash sign
|
|
|
+\(aq#\(aq after removing whitespace on both ends are ignored. The optional style
|
|
|
selector prefix is also supported for patterns loaded from a file. Due to
|
|
|
whitespace removal, paths with whitespace at the beginning or end can only be
|
|
|
excluded using regular expressions.
|
|
@@ -155,74 +152,101 @@ Examples:
|
|
|
.nf
|
|
|
.ft C
|
|
|
# Exclude \(aq/home/user/file.o\(aq but not \(aq/home/user/file.odt\(aq:
|
|
|
-$ borg create \-e \(aq*.o\(aq backup /
|
|
|
+$ borg create \-e \(aq*.o\(aq archive /
|
|
|
|
|
|
# Exclude \(aq/home/user/junk\(aq and \(aq/home/user/subdir/junk\(aq but
|
|
|
# not \(aq/home/user/importantjunk\(aq or \(aq/etc/junk\(aq:
|
|
|
-$ borg create \-e \(aq/home/*/junk\(aq backup /
|
|
|
+$ borg create \-e \(aqhome/*/junk\(aq archive /
|
|
|
|
|
|
# Exclude the contents of \(aq/home/user/cache\(aq but not the directory itself:
|
|
|
-$ borg create \-e home/user/cache/ backup /
|
|
|
+$ borg create \-e home/user/cache/ archive /
|
|
|
|
|
|
# The file \(aq/home/user/cache/important\(aq is *not* backed up:
|
|
|
-$ borg create \-e /home/user/cache/ backup / /home/user/cache/important
|
|
|
+$ borg create \-e home/user/cache/ archive / /home/user/cache/important
|
|
|
|
|
|
# The contents of directories in \(aq/home\(aq are not backed up when their name
|
|
|
# ends in \(aq.tmp\(aq
|
|
|
-$ borg create \-\-exclude \(aqre:^/home/[^/]+\e.tmp/\(aq backup /
|
|
|
+$ borg create \-\-exclude \(aqre:^home/[^/]+\e.tmp/\(aq archive /
|
|
|
|
|
|
# Load exclusions from file
|
|
|
$ cat >exclude.txt <<EOF
|
|
|
# Comment line
|
|
|
-/home/*/junk
|
|
|
+home/*/junk
|
|
|
*.tmp
|
|
|
fm:aa:something/*
|
|
|
-re:^/home/[^/]+\e.tmp/
|
|
|
-sh:/home/*/.thumbnails
|
|
|
+re:^home/[^/]+\e.tmp/
|
|
|
+sh:home/*/.thumbnails
|
|
|
# Example with spaces, no need to escape as it is processed by borg
|
|
|
some file with spaces.txt
|
|
|
EOF
|
|
|
-$ borg create \-\-exclude\-from exclude.txt backup /
|
|
|
+$ borg create \-\-exclude\-from exclude.txt archive /
|
|
|
.ft P
|
|
|
.fi
|
|
|
.UNINDENT
|
|
|
.UNINDENT
|
|
|
.sp
|
|
|
-A more general and easier to use way to define filename matching patterns exists
|
|
|
-with the \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP options. Using these, you may
|
|
|
-specify the backup roots (starting points) and patterns for inclusion/exclusion.
|
|
|
-A root path starts with the prefix \fIR\fP, followed by a path (a plain path, not a
|
|
|
-file pattern). An include rule starts with the prefix +, an exclude rule starts
|
|
|
-with the prefix \-, an exclude\-norecurse rule starts with !, all followed by a pattern.
|
|
|
+A more general and easier to use way to define filename matching patterns
|
|
|
+exists with the \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP options. Using
|
|
|
+these, you may specify the backup roots, default pattern styles and
|
|
|
+patterns for inclusion and exclusion.
|
|
|
+.INDENT 0.0
|
|
|
+.TP
|
|
|
+.B Root path prefix \fBR\fP
|
|
|
+A recursion root path starts with the prefix \fBR\fP, followed by a path
|
|
|
+(a plain path, not a file pattern). Use this prefix to have the root
|
|
|
+paths in the patterns file rather than as command line arguments.
|
|
|
+.TP
|
|
|
+.B Pattern style prefix \fBP\fP
|
|
|
+To change the default pattern style, use the \fBP\fP prefix, followed by
|
|
|
+the pattern style abbreviation (\fBfm\fP, \fBpf\fP, \fBpp\fP, \fBre\fP, \fBsh\fP).
|
|
|
+All patterns following this line will use this style until another style
|
|
|
+is specified.
|
|
|
+.TP
|
|
|
+.B Exclude pattern prefix \fB\-\fP
|
|
|
+Use the prefix \fB\-\fP, followed by a pattern, to define an exclusion.
|
|
|
+This has the same effect as the \fB\-\-exclude\fP option.
|
|
|
+.TP
|
|
|
+.B Exclude no\-recurse pattern prefix \fB!\fP
|
|
|
+Use the prefix \fB!\fP, followed by a pattern, to define an exclusion
|
|
|
+that does not recurse into subdirectories. This saves time, but
|
|
|
+prevents include patterns to match any files in subdirectories.
|
|
|
+.TP
|
|
|
+.B Include pattern prefix \fB+\fP
|
|
|
+Use the prefix \fB+\fP, followed by a pattern, to define inclusions.
|
|
|
+This is useful to include paths that are covered in an exclude
|
|
|
+pattern and would otherwise not be backed up.
|
|
|
+.UNINDENT
|
|
|
+.sp
|
|
|
+The first matching pattern is used, so if an include pattern matches
|
|
|
+before an exclude pattern, the file is backed up. Note that a no\-recurse
|
|
|
+exclude stops examination of subdirectories so that potential includes
|
|
|
+will not match \- use normal exludes for such use cases.
|
|
|
.sp
|
|
|
-\fBNOTE:\fP
|
|
|
+\fBTip: You can easily test your patterns with \-\-dry\-run and \-\-list\fP:
|
|
|
.INDENT 0.0
|
|
|
.INDENT 3.5
|
|
|
-Via \fB\-\-pattern\fP or \fB\-\-patterns\-from\fP you can define BOTH inclusion and exclusion
|
|
|
-of files using pattern prefixes \fB+\fP and \fB\-\fP\&. With \fB\-\-exclude\fP and
|
|
|
-\fB\-\-exclude\-from\fP ONLY excludes are defined.
|
|
|
+.sp
|
|
|
+.nf
|
|
|
+.ft C
|
|
|
+$ borg create \-\-dry\-run \-\-list \-\-patterns\-from patterns.txt archive
|
|
|
+.ft P
|
|
|
+.fi
|
|
|
.UNINDENT
|
|
|
.UNINDENT
|
|
|
.sp
|
|
|
-Inclusion patterns are useful to include paths that are contained in an excluded
|
|
|
-path. The first matching pattern is used so if an include pattern matches before
|
|
|
-an exclude pattern, the file is backed up. If an exclude\-norecurse pattern matches
|
|
|
-a directory, it won\(aqt recurse into it and won\(aqt discover any potential matches for
|
|
|
-include rules below that directory.
|
|
|
+This will list the considered files one per line, prefixed with a
|
|
|
+character that indicates the action (e.g. \(aqx\(aq for excluding, see
|
|
|
+\fBItem flags\fP in \fIborg create\fP usage docs).
|
|
|
.sp
|
|
|
\fBNOTE:\fP
|
|
|
.INDENT 0.0
|
|
|
.INDENT 3.5
|
|
|
-It\(aqs possible that a sub\-directory/file is matched while parent directories are not.
|
|
|
-In that case, parent directories are not backed up thus their user, group, permission,
|
|
|
-etc. can not be restored.
|
|
|
+It\(aqs possible that a sub\-directory/file is matched while parent
|
|
|
+directories are not. In that case, parent directories are not backed
|
|
|
+up and thus their user, group, permission, etc. cannot be restored.
|
|
|
.UNINDENT
|
|
|
.UNINDENT
|
|
|
.sp
|
|
|
-Note that the default pattern style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP is
|
|
|
-shell style (\fIsh:\fP), so those patterns behave similar to rsync include/exclude
|
|
|
-patterns. The pattern style can be set via the \fIP\fP prefix.
|
|
|
-.sp
|
|
|
Patterns (\fB\-\-pattern\fP) and excludes (\fB\-\-exclude\fP) from the command line are
|
|
|
considered first (in the order of appearance). Then patterns from \fB\-\-patterns\-from\fP
|
|
|
are added. Exclusion patterns from \fB\-\-exclude\-from\fP files are appended last.
|
|
@@ -235,10 +259,16 @@ Examples:
|
|
|
.ft C
|
|
|
# backup pics, but not the ones from 2018, except the good ones:
|
|
|
# note: using = is essential to avoid cmdline argument parsing issues.
|
|
|
-borg create \-\-pattern=+pics/2018/good \-\-pattern=\-pics/2018 repo::arch pics
|
|
|
+borg create \-\-pattern=+pics/2018/good \-\-pattern=\-pics/2018 archive pics
|
|
|
|
|
|
-# use a file with patterns:
|
|
|
-borg create \-\-patterns\-from patterns.lst repo::arch
|
|
|
+# backup only JPG/JPEG files (case insensitive) in all home directories:
|
|
|
+borg create \-\-pattern \(aq+ re:\e.jpe?g(?i)$\(aq archive /home
|
|
|
+
|
|
|
+# backup homes, but exclude big downloads (like .ISO files) or hidden files:
|
|
|
+borg create \-\-exclude \(aqre:\e.iso(?i)$\(aq \-\-exclude \(aqsh:home/**/.*\(aq archive /home
|
|
|
+
|
|
|
+# use a file with patterns (recursion root \(aq/\(aq via command line):
|
|
|
+borg create \-\-patterns\-from patterns.lst archive /
|
|
|
.ft P
|
|
|
.fi
|
|
|
.UNINDENT
|
|
@@ -250,22 +280,17 @@ The patterns.lst file could look like that:
|
|
|
.sp
|
|
|
.nf
|
|
|
.ft C
|
|
|
-# "sh:" pattern style is the default, so the following line is not needed:
|
|
|
-P sh
|
|
|
-R /
|
|
|
-# can be rebuild
|
|
|
-\- /home/*/.cache
|
|
|
-# they\(aqre downloads for a reason
|
|
|
-\- /home/*/Downloads
|
|
|
-# susan is a nice person
|
|
|
+# "sh:" pattern style is the default
|
|
|
+# exclude caches
|
|
|
+\- home/*/.cache
|
|
|
# include susans home
|
|
|
-+ /home/susan
|
|
|
++ home/susan
|
|
|
# also back up this exact file
|
|
|
-+ pf:/home/bobby/specialfile.txt
|
|
|
++ pf:home/bobby/specialfile.txt
|
|
|
# don\(aqt backup the other home directories
|
|
|
-\- /home/*
|
|
|
-# don\(aqt even look in /proc
|
|
|
-! /proc
|
|
|
+\- home/*
|
|
|
+# don\(aqt even look in /dev, /proc, /run, /sys, /tmp (note: would exclude files like /device, too)
|
|
|
+! re:^(dev|proc|run|sys|tmp)
|
|
|
.ft P
|
|
|
.fi
|
|
|
.UNINDENT
|
|
@@ -278,14 +303,14 @@ You can specify recursion roots either on the command line or in a patternfile:
|
|
|
.nf
|
|
|
.ft C
|
|
|
# these two commands do the same thing
|
|
|
-borg create \-\-exclude /home/bobby/junk repo::arch /home/bobby /home/susan
|
|
|
-borg create \-\-patterns\-from patternfile.lst repo::arch
|
|
|
+borg create \-\-exclude home/bobby/junk archive /home/bobby /home/susan
|
|
|
+borg create \-\-patterns\-from patternfile.lst archive
|
|
|
.ft P
|
|
|
.fi
|
|
|
.UNINDENT
|
|
|
.UNINDENT
|
|
|
.sp
|
|
|
-The patternfile:
|
|
|
+patternfile.lst:
|
|
|
.INDENT 0.0
|
|
|
.INDENT 3.5
|
|
|
.sp
|
|
@@ -296,8 +321,7 @@ The patternfile:
|
|
|
P fm
|
|
|
R /home/bobby
|
|
|
R /home/susan
|
|
|
-
|
|
|
-\- /home/bobby/junk
|
|
|
+\- home/bobby/junk
|
|
|
.ft P
|
|
|
.fi
|
|
|
.UNINDENT
|