Backup to a removable drive or intermittent server via "soft failure" feature (#284).

NEWS

@@ -3,6 +3,9 @@
  * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
  * #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
    should be excluded from backups, rather than just a single filename.
+ * #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
  * #287: View consistency check progress via "--progress" flag for "check" action.
  * For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
    by default. You can opt back in with "--files" or "--stats" flags.

README.md

@@ -24,7 +24,7 @@ location:
     repositories:
         - 1234@usw-s001.rsync.net:backups.borg
         - k8pDxu32@k8pDxu32.repo.borgbase.com:repo
-        - /var/lib/backups/backups.borg
+        - /var/lib/backups/local.borg
 
 retention:
     # Retention policy for how many backups to keep.
@@ -80,6 +80,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
  * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
  * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
  * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/)
  * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
  * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 

borgmatic/commands/borgmatic.py

@@ -83,6 +83,9 @@ def run_configuration(config_filename, config, arguments):
                 global_arguments.dry_run,
             )
     except (OSError, CalledProcessError) as error:
+        if command.considered_soft_failure(config_filename, error):
+            return
+
         encountered_error = error
         yield from make_error_log_records(
             '{}: Error running pre-backup hook'.format(config_filename), error
@@ -138,6 +141,9 @@ def run_configuration(config_filename, config, arguments):
                     global_arguments.dry_run,
                 )
         except (OSError, CalledProcessError) as error:
+            if command.considered_soft_failure(config_filename, error):
+                return
+
             encountered_error = error
             yield from make_error_log_records(
                 '{}: Error running post-backup hook'.format(config_filename), error
@@ -165,6 +171,9 @@ def run_configuration(config_filename, config, arguments):
                 global_arguments.dry_run,
             )
         except (OSError, CalledProcessError) as error:
+            if command.considered_soft_failure(config_filename, error):
+                return
+
             yield from make_error_log_records(
                 '{}: Error running on-error hook'.format(config_filename), error
             )

borgmatic/config/schema.yaml

@@ -548,7 +548,8 @@ map:
                     - type: str
                 desc: |
                     List of one or more shell commands or scripts to execute before running all
-                    actions (if one of them is "create"), run once before all configuration files.
+                    actions (if one of them is "create"). These are collected from all configuration
+                    files and then run once before all of them (prior to all actions).
                 example:
                     - echo "Starting actions."
             after_everything:
@@ -556,7 +557,8 @@ map:
                     - type: str
                 desc: |
                     List of one or more shell commands or scripts to execute after running all
-                    actions (if one of them is "create"), run once after all configuration files.
+                    actions (if one of them is "create"). These are collected from all configuration
+                    files and then run once before all of them (prior to all actions).
                 example:
                     - echo "Completed actions."
             umask:

borgmatic/hooks/command.py

@@ -6,6 +6,9 @@ from borgmatic import execute
 logger = logging.getLogger(__name__)
 
 
+SOFT_FAIL_EXIT_CODE = 75
+
+
 def interpolate_context(command, context):
     '''
     Given a single hook command and a dict of context names/values, interpolate the values by
@@ -69,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
     finally:
         if original_umask:
             os.umask(original_umask)
+
+
+def considered_soft_failure(config_filename, error):
+    '''
+    Given a configuration filename and an exception object, return whether the exception object
+    represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
+    that indicates that the error is a "soft failure", and should not result in an error.
+    '''
+    exit_code = getattr(error, 'returncode', None)
+    if exit_code is None:
+        return False
+
+    if exit_code == SOFT_FAIL_EXIT_CODE:
+        logger.info(
+            '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
+                config_filename, SOFT_FAIL_EXIT_CODE
+            )
+        )
+        return True
+
+    return False

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -0,0 +1,106 @@
+---
+title: Backup to a removable drive or an intermittent server
+---
+## Occasional backups
+
+A common situation is backing up to a repository that's only sometimes online.
+For instance, you might send most of your backups to the cloud, but
+occasionally you want to plug in an external hard drive or backup to your
+buddy's sometimes-online server for that extra level of redundancy.
+
+But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
+server is offline, then you'll get an annoying error message and the overall
+borgmatic run will fail (even if individual repositories complete just fine).
+
+So what if you want borgmatic to swallow the error of a missing drive
+or an offline server, and continue trucking along? That's where the concept of
+"soft failure" come in.
+
+## Soft failure command hooks
+
+This feature leverages [borgmatic command
+hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
+so first familiarize yourself with them. The idea is that you write a simple
+test in the form of a borgmatic hook to see if backups should proceed or not.
+
+The way the test works is that if any of your hook commands return a special
+exit status of 75, that indicates to borgmatic that it's a temporary failure,
+and borgmatic should skip all subsequent actions for that configuration file.
+If you return any other status, then it's a standard success or error. (Zero is
+success; anything else other than 75 is an error).
+
+So for instance, if you have an external drive that's only sometimes mounted,
+declare its repository in its own [separate configuration
+file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/),
+say at `/etc/borgmatic.d/removable.yaml`:
+
+```yaml
+location:
+    source_directories:
+        - /home
+
+    repositories:
+        - /mnt/removable/backup.borg
+```
+
+Then, write a `before_backup` hook in that same configuration file that uses
+the external `findmnt` utility to see whether the drive is mounted before
+proceeding.
+
+```yaml
+hooks:
+    before_backup:
+      - findmnt /mnt/removable > /dev/null || exit 75
+```
+
+What this does is check if the `findmnt` command errors when probing for a
+particular mount point. If it does error, then it returns exit code 75 to
+borgmatic. borgmatic logs the soft failure, skips all further actions in that
+configurable file, and proceeds onward to any other borgmatic configuration
+files you may have.
+
+You can imagine a similar check for the sometimes-online server case:
+
+```yaml
+location:
+    source_directories:
+        - /home
+
+    repositories:
+        - me@buddys-server.org:backup.borg
+
+hooks:
+    before_backup:
+      - ping -q -c 1 buddys-server.org > /dev/null || exit 75
+```
+
+## Caveats and details
+
+There are some caveats you should be aware of with this feature.
+
+ * You'll generally want to put a soft failure command in the `before_backup`
+   hook, so as to gate whether the backup action occurs. While a soft failure is
+   also supported in the `after_backup` hook, returning a soft failure there
+   won't prevent any actions from occuring, because they've already occurred!
+   Similiarly, you can return a soft failure from an `on_error` hook, but at
+   that point it's too late to prevent the error.
+ * Returning a soft failure does prevent further commands in the same hook from
+   executing. So, like a standard error, it is an "early out". Unlike a standard
+   error, borgmatic does not display it in angry red text or consider it a
+   failure.
+ * The soft failure only applies to the scope of a single borgmatic
+   configuration file. So put anything that you don't want soft-failed, like
+   always-online cloud backups, in separate configuration files from your
+   soft-failing repositories.
+ * The soft failure doesn't have to apply to a repository. You can even perform
+   a test to make sure that individual source directories are mounted and
+   available. Use your imagination!
+ * This feature does not apply to `before_everything` or `after_everything`
+   hooks.
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
+ * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)

tests/unit/commands/test_borgmatic.py

@@ -3,6 +3,7 @@ import subprocess
 
 from flexmock import flexmock
 
+import borgmatic.hooks.command
 from borgmatic.commands import borgmatic as module
 
 
@@ -93,6 +94,20 @@ def test_run_configuration_logs_pre_hook_error():
     assert results == expected_results
 
 
+def test_run_configuration_bails_for_pre_hook_soft_failure():
+    flexmock(module.borg_environment).should_receive('initialize')
+    error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
+    flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None)
+    flexmock(module).should_receive('make_error_log_records').never()
+    flexmock(module).should_receive('run_actions').never()
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == []
+
+
 def test_run_configuration_logs_post_hook_error():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
@@ -110,6 +125,23 @@ def test_run_configuration_logs_post_hook_error():
     assert results == expected_results
 
 
+def test_run_configuration_bails_for_post_hook_soft_failure():
+    flexmock(module.borg_environment).should_receive('initialize')
+    error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
+    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
+        error
+    ).and_return(None)
+    flexmock(module.dispatch).should_receive('call_hooks')
+    flexmock(module).should_receive('make_error_log_records').never()
+    flexmock(module).should_receive('run_actions').and_return([])
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == []
+
+
 def test_run_configuration_logs_on_error_hook_error():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
@@ -126,6 +158,21 @@ def test_run_configuration_logs_on_error_hook_error():
     assert results == expected_results
 
 
+def test_run_configuration_bails_for_on_error_hook_soft_failure():
+    flexmock(module.borg_environment).should_receive('initialize')
+    error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
+    flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error)
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
+    flexmock(module).should_receive('run_actions').and_raise(OSError)
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+
+    results = list(module.run_configuration('test.yaml', config, arguments))
+
+    assert results == expected_results
+
+
 def test_load_configurations_collects_parsed_configurations():
     configuration = flexmock()
     other_configuration = flexmock()

tests/unit/hooks/test_command.py

@@ -1,4 +1,5 @@
 import logging
+import subprocess
 
 from flexmock import flexmock
 
@@ -79,3 +80,19 @@ def test_execute_hook_on_error_logs_as_error():
     ).once()
 
     module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
+
+
+def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail():
+    error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again')
+
+    assert module.considered_soft_failure('config.yaml', error)
+
+
+def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail():
+    error = subprocess.CalledProcessError(1, 'error')
+
+    assert not module.considered_soft_failure('config.yaml', error)
+
+
+def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail():
+    assert not module.considered_soft_failure('config.yaml', Exception())