Merge pull request #1194 from ThomasWaldmann/more-placeholders

add placeholder support at missing places, add missing help

borg/archiver.py

@@ -20,7 +20,7 @@ from .helpers import Error, location_validator, archivename_validator, format_li
     parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, \
     get_cache_dir, prune_within, prune_split, \
     Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \
+    dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher
 from .logger import create_logger, setup_logging
 logger = create_logger()
@@ -773,35 +773,68 @@ class Archiver:
         whitespace removal paths with whitespace at the beginning or end can only be
         excluded using regular expressions.
 
-        Examples:
+        Examples::
 
-        # Exclude '/home/user/file.o' but not '/home/user/file.odt':
-        $ borg create -e '*.o' backup /
+            # Exclude '/home/user/file.o' but not '/home/user/file.odt':
+            $ borg create -e '*.o' backup /
 
-        # Exclude '/home/user/junk' and '/home/user/subdir/junk' but
-        # not '/home/user/importantjunk' or '/etc/junk':
-        $ borg create -e '/home/*/junk' backup /
+            # Exclude '/home/user/junk' and '/home/user/subdir/junk' but
+            # not '/home/user/importantjunk' or '/etc/junk':
+            $ borg create -e '/home/*/junk' backup /
 
-        # Exclude the contents of '/home/user/cache' but not the directory itself:
-        $ borg create -e /home/user/cache/ backup /
+            # Exclude the contents of '/home/user/cache' but not the directory itself:
+            $ borg create -e /home/user/cache/ backup /
 
-        # The file '/home/user/cache/important' is *not* backed up:
-        $ borg create -e /home/user/cache/ backup / /home/user/cache/important
+            # The file '/home/user/cache/important' is *not* backed up:
+            $ borg create -e /home/user/cache/ backup / /home/user/cache/important
 
-        # The contents of directories in '/home' are not backed up when their name
-        # ends in '.tmp'
-        $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
+            # The contents of directories in '/home' are not backed up when their name
+            # ends in '.tmp'
+            $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup /
 
-        # Load exclusions from file
-        $ cat >exclude.txt <<EOF
-        # Comment line
-        /home/*/junk
-        *.tmp
-        fm:aa:something/*
-        re:^/home/[^/]\.tmp/
-        sh:/home/*/.thumbnails
-        EOF
-        $ borg create --exclude-from exclude.txt backup /
+            # Load exclusions from file
+            $ cat >exclude.txt <<EOF
+            # Comment line
+            /home/*/junk
+            *.tmp
+            fm:aa:something/*
+            re:^/home/[^/]\.tmp/
+            sh:/home/*/.thumbnails
+            EOF
+            $ borg create --exclude-from exclude.txt backup /
+        ''')
+    helptext['placeholders'] = textwrap.dedent('''
+        Repository (or Archive) URLs and --prefix values support these placeholders:
+
+        {hostname}
+
+            The (short) hostname of the machine.
+
+        {fqdn}
+
+            The full name of the machine.
+
+        {now}
+
+            The current local date and time.
+
+        {utcnow}
+
+            The current UTC date and time.
+
+        {user}
+
+            The user name (or UID, if no name is available) of the user running borg.
+
+        {pid}
+
+            The current process ID.
+
+        Examples::
+
+            borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
+            borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
+            borg prune --prefix '{hostname}-' ...
         ''')
 
     def do_help(self, parser, commands, args):
@@ -952,7 +985,7 @@ class Archiver:
         subparser.add_argument('--last', dest='last',
                                type=int, default=None, metavar='N',
                                help='only check last N archives (Default: all)')
-        subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
+        subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
                                help='only consider archive names starting with this prefix')
 
         change_passphrase_epilog = textwrap.dedent("""
@@ -1013,6 +1046,7 @@ class Archiver:
         all files on these file systems.
 
         See the output of the "borg help patterns" command for more help on exclude patterns.
+        See the output of the "borg help placeholders" command for more help on placeholders.
         """)
 
         subparser = subparsers.add_parser('create', parents=[common_parser],
@@ -1194,7 +1228,7 @@ class Archiver:
                                help="""specify format for archive file listing
                                 (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}")
                                 Special "{formatkeys}" exists to list available keys""")
-        subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
+        subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
                                help='only consider archive names starting with this prefix')
         subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
                                type=location_validator(),
@@ -1301,7 +1335,7 @@ class Archiver:
                                help='number of monthly archives to keep')
         subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0,
                                help='number of yearly archives to keep')
-        subparser.add_argument('-P', '--prefix', dest='prefix', type=str,
+        subparser.add_argument('-P', '--prefix', dest='prefix', type=PrefixSpec,
                                help='only consider archive names starting with this prefix')
         subparser.add_argument('--save-space', dest='save_space', action='store_true',
                                default=False,
@@ -1522,7 +1556,15 @@ def main():  # pragma: no cover
     setup_signal_handlers()
     archiver = Archiver()
     msg = None
-    args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
+    try:
+        args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND'))
+    except Error as e:
+        msg = e.get_message()
+        if e.traceback:
+            msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo())
+        # we might not have logging setup yet, so get out quickly
+        print(msg, file=sys.stderr)
+        sys.exit(e.exit_code)
     try:
         exit_code = archiver.run(args)
     except Error as e:

borg/helpers.py

@@ -69,6 +69,10 @@ class NoManifestError(Error):
     """Repository has no manifest."""
 
 
+class PlaceholderError(Error):
+    """Formatting Error: "{}".format({}): {}({})"""
+
+
 def check_extension_modules():
     from . import platform
     if hashindex.API_VERSION != 2:
@@ -514,6 +518,10 @@ def CompressionSpec(s):
     raise ValueError
 
 
+def PrefixSpec(s):
+    return replace_placeholders(s)
+
+
 def dir_is_cachedir(path):
     """Determines whether the specified path is a cache directory (and
     therefore should potentially be excluded from the backup) according to
@@ -552,18 +560,24 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
 
 
 def format_line(format, data):
-    # TODO: Filter out unwanted properties of str.format(), because "format" is user provided.
-
     try:
         return format.format(**data)
-    except (KeyError, ValueError) as e:
-        # this should catch format errors
-        print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
     except Exception as e:
-        # something unexpected, print error and raise exception
-        print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e)))
-        raise
-    return ''
+        raise PlaceholderError(format, data, e.__class__.__name__, str(e))
+
+
+def replace_placeholders(text):
+    """Replace placeholders in text with their values."""
+    current_time = datetime.now()
+    data = {
+        'pid': os.getpid(),
+        'fqdn': socket.getfqdn(),
+        'hostname': socket.gethostname(),
+        'now': current_time.now(),
+        'utcnow': current_time.utcnow(),
+        'user': uid2user(os.getuid(), os.getuid())
+    }
+    return format_line(text, data)
 
 
 def safe_timestamp(item_timestamp_ns):
@@ -720,21 +734,8 @@ class Location:
         if not self.parse(self.orig):
             raise ValueError
 
-    def preformat_text(self, text):
-        """Format repository and archive path with common tags"""
-        current_time = datetime.now()
-        data = {
-            'pid': os.getpid(),
-            'fqdn': socket.getfqdn(),
-            'hostname': socket.gethostname(),
-            'now': current_time.now(),
-            'utcnow': current_time.utcnow(),
-            'user': uid2user(os.getuid(), os.getuid())
-            }
-        return format_line(text, data)
-
     def parse(self, text):
-        text = self.preformat_text(text)
+        text = replace_placeholders(text)
         valid = self._parse(text)
         if valid:
             return True

borg/testsuite/archiver.py

@@ -946,7 +946,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.cmd('create', test_archive, src_dir)
         output_1 = self.cmd('list', test_archive)
         output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive)
-        output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive)
+        output_3 = self.cmd('list', '--list-format', '{mtime:%s} {path}{NEWLINE}', test_archive)
         self.assertEqual(output_1, output_2)
         self.assertNotEqual(output_1, output_3)
 

borg/testsuite/helpers.py

@@ -10,7 +10,7 @@ import msgpack
 import msgpack.fallback
 import time
 
-from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \
+from ..helpers import Location, format_file_size, format_timedelta, format_line, PlaceholderError, make_path_safe, \
     prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \
     yes, TRUISH, FALSISH, DEFAULTISH, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \
@@ -877,3 +877,18 @@ def test_progress_endless_step(capfd):
     pi.show()
     out, err = capfd.readouterr()
     assert err == '.'
+
+
+def test_format_line():
+    data = dict(foo='bar baz')
+    assert format_line('', data) == ''
+    assert format_line('{foo}', data) == 'bar baz'
+    assert format_line('foo{foo}foo', data) == 'foobar bazfoo'
+
+
+def test_format_line_erroneous():
+    data = dict()
+    with pytest.raises(PlaceholderError):
+        assert format_line('{invalid}', data)
+    with pytest.raises(PlaceholderError):
+        assert format_line('{}', data)

docs/quickstart.rst

@@ -110,7 +110,7 @@ certain number of old archives::
     # Backup all of /home and /var/www except a few
     # excluded directories
     borg create -v --stats                          \
-        $REPOSITORY::`hostname`-`date +%Y-%m-%d`    \
+        $REPOSITORY::'{hostname}-{now:%Y-%m-%d}'    \
         /home                                       \
         /var/www                                    \
         --exclude '/home/*/.cache'                  \
@@ -118,10 +118,10 @@ certain number of old archives::
         --exclude '*.pyc'
 
     # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
-    # archives of THIS machine. --prefix `hostname`- is very important to
+    # archives of THIS machine. The '{hostname}-' prefix is very important to
     # limit prune's operation to this machine's archives and not apply to
     # other machine's archives also.
-    borg prune -v $REPOSITORY --prefix `hostname`- \
+    borg prune -v $REPOSITORY --prefix '{hostname}-' \
         --keep-daily=7 --keep-weekly=4 --keep-monthly=6
 
 .. backup_compression:

docs/usage.rst

@@ -425,8 +425,9 @@ will see what it would do without it actually doing anything.
     # Do a dry-run without actually deleting anything.
     $ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /path/to/repo
 
-    # Same as above but only apply to archive names starting with "foo":
-    $ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /path/to/repo
+    # Same as above but only apply to archive names starting with the hostname
+    # of the machine followed by a "-" character:
+    $ borg prune --keep-daily=7 --keep-weekly=4 --prefix='{hostname}-' /path/to/repo
 
     # Keep 7 end of day, 4 additional end of week archives,
     # and an end of month archive for every month: