Initial work on a Btrfs hook (#251).

NEWS

@@ -1,4 +1,7 @@
 1.9.4.dev0
+ * #251 (beta): Add a Btrfs hook for snapshotting and backing up Btrfs subvolumes. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/snapshot-your-filesystems/
  * #926: Fix library error when running within a PyInstaller bundle.
  * Reorganize data source and monitoring hooks to make developing new hooks easier.
 

borgmatic/config/paths.py

@@ -33,10 +33,11 @@ def get_borgmatic_source_directory(config):
 TEMPORARY_DIRECTORY_PREFIX = 'borgmatic-'
 
 
-def replace_temporary_subdirectory_with_glob(path):
+def replace_temporary_subdirectory_with_glob(path, temporary_directory_prefix=TEMPORARY_DIRECTORY_PREFIX):
     '''
-    Given an absolute temporary directory path, look for a subdirectory within it starting with the
-    temporary directory prefix and replace it with an appropriate glob. For instance, given:
+    Given an absolute temporary directory path and an optional temporary directory prefix, look for
+    a subdirectory within it starting with the temporary directory prefix (or a default) and replace
+    it with an appropriate glob. For instance, given:
 
         /tmp/borgmatic-aet8kn93/borgmatic
 
@@ -50,8 +51,8 @@ def replace_temporary_subdirectory_with_glob(path):
         '/',
         *(
             (
-                f'{TEMPORARY_DIRECTORY_PREFIX}*'
-                if subdirectory.startswith(TEMPORARY_DIRECTORY_PREFIX)
+                f'{temporary_directory_prefix}*'
+                if subdirectory.startswith(temporary_directory_prefix)
                 else subdirectory
             )
             for subdirectory in path.split(os.path.sep)

borgmatic/config/schema.yaml

@@ -2288,3 +2288,17 @@ properties:
                 example: /usr/local/bin/umount
         description: |
             Configuration for integration with the ZFS filesystem.
+    btrfs:
+        type: ["object", "null"]
+        additionalProperties: false
+        properties:
+            btrfs_command:
+                type: string
+                description: |
+                    Command to use instead of "btrfs".
+                example: /usr/local/bin/btrfs
+            findmnt_command:
+                type: string
+                description: |
+                    Command to use instead of "findmnt".
+                example: /usr/local/bin/findmnt

borgmatic/hooks/data_source/btrfs.py

@@ -0,0 +1,283 @@
+import functools
+import glob
+import logging
+import os
+import shutil
+import subprocess
+
+import borgmatic.config.paths
+import borgmatic.execute
+
+logger = logging.getLogger(__name__)
+
+
+def use_streaming(hook_config, config, log_prefix):  # pragma: no cover
+    '''
+    Return whether dump streaming is used for this hook. (Spoiler: It isn't.)
+    '''
+    return False
+
+
+def get_subvolumes(btrfs_command, findmnt_command, source_directories=None):
+    '''
+    Given a Btrfs command to run and a sequence of configured source directories, find the
+    intersection between the current Btrfs filesystem and subvolume mount points and the configured
+    borgmatic source directories. The idea is that these are the requested subvolumes to snapshot.
+
+    If the source directories is None, then return all subvolumes.
+
+    Return the result as a sequence of matching subvolume mount points.
+    '''
+    # Find all top-level Btrfs filesystem mount points.
+    findmnt_output = borgmatic.execute.execute_command_and_capture_output(
+        (
+            findmnt_command,
+            '-nt',
+            'btrfs',
+        )
+    )
+
+    try:
+        filesystem_mount_points = tuple(
+            line.rstrip().split(' ')[0]
+            for line in findmnt_output.splitlines()
+        )
+    except ValueError:
+        raise ValueError('Invalid {findmnt_command} output')
+
+    source_directories_set = set(source_directories or ())
+    subvolumes = []
+
+    # For each filesystem mount point, find its subvolumes and match them again the given source
+    # directories to find the subvolumes to backup. Also try to match the filesystem mount point
+    # itself.
+    for mount_point in filesystem_mount_points:
+        if source_directories is None or mount_point in source_directories_set:
+            subvolumes.append(mount_point)
+
+        btrfs_output = borgmatic.execute.execute_command_and_capture_output(
+            (
+                btrfs_command,
+                'subvolume',
+                'list',
+                mount_point,
+            )
+        )
+
+        try:
+            subvolumes.extend(
+                subvolume_path
+                for line in btrfs_output.splitlines()
+                for subvolume_subpath in (line.rstrip().split(' ')[-1],)
+                for subvolume_path in (os.path.join(mount_point, subvolume_subpath),)
+                if source_directories is None or subvolume_path in source_directories_set
+            )
+        except ValueError:
+            raise ValueError('Invalid {btrfs_command} subvolume list output')
+
+    return tuple(subvolumes)
+
+
+BORGMATIC_SNAPSHOT_PREFIX = '.borgmatic-snapshot-'
+
+
+def make_snapshot_path(subvolume_path):  # pragma: no cover
+    '''
+    Given the path to a subvolume, make a corresponding snapshot path for it.
+    '''
+    return os.path.join(
+        subvolume_path,
+        f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
+        '.',  # Borg 1.4+ "slashdot" hack.
+        # Included so that the snapshot ends up in the Borg archive at the "original" subvolume
+        # path.
+        subvolume_path.lstrip(os.path.sep),
+    )
+
+
+def make_snapshot_exclude_path(subvolume_path):  # pragma: no cover
+    '''
+    Given the path to a subvolume, make a corresponding exclude path for its embedded snapshot path.
+    This is to work around a quirk of Btrfs: If you make a snapshot path as a child directory of a
+    subvolume, then the snapshot's own initial directory component shows up as an empty directory
+    within the snapshot itself. For instance, if you have a Btrfs subvolume at /mnt and make a
+    snapshot of it at:
+
+        /mnt/.borgmatic-snapshot-1234/mnt
+
+    ... then the snapshot itself will have an empty directory at:
+
+        /mnt/.borgmatic-snapshot-1234/mnt/.borgmatic-snapshot-1234
+
+    So to prevent that from ending up in the Borg archive, this function produces its path for
+    exclusion.
+    '''
+    snapshot_directory = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}'
+
+    return os.path.join(
+        subvolume_path,
+        snapshot_directory,
+        subvolume_path.lstrip(os.path.sep),
+        snapshot_directory,
+    )
+
+
+def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path):  # pragma: no cover
+    '''
+    Given a Btrfs command to run, the path to a subvolume, and the path for a snapshot, create a new
+    Btrfs snapshot of the subvolume.
+    '''
+    os.makedirs(os.path.dirname(snapshot_path), mode=0o700, exist_ok=True)
+
+    borgmatic.execute.execute_command(
+        (
+            btrfs_command,
+            'subvolume',
+            'snapshot',
+            subvolume_path,
+            snapshot_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def dump_data_sources(
+    hook_config,
+    config,
+    log_prefix,
+    config_paths,
+    borgmatic_runtime_directory,
+    source_directories,
+    dry_run,
+):
+    '''
+    Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic
+    configuration file paths, the borgmatic runtime directory, the configured source directories,
+    and whether this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed
+    in the given source directories. Also update those source directories, replacing subvolume mount
+    points with corresponding snapshot directories so they get stored in the Borg archive instead.
+    Use the log prefix in any log entries.
+
+    Return an empty sequence, since there are no ongoing dump processes from this hook.
+
+    If this is a dry run, then don't actually snapshot anything.
+    '''
+    dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else ''
+    logger.info(f'{log_prefix}: Snapshotting Btrfs datasets{dry_run_label}')
+
+    # Based on the configured source directories, determine Btrfs subvolumes to backup.
+    btrfs_command = hook_config.get('btrfs_command', 'btrfs')
+    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
+    requested_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command, source_directories)
+
+    # Snapshot each subvolume, rewriting source directories to use their snapshot paths.
+    for subvolume_path in requested_subvolume_paths:
+        logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume_path} subvolume')
+
+        snapshot_path = make_snapshot_path(subvolume_path)
+
+        if dry_run:
+            continue
+
+        snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path)
+
+        if subvolume_path in source_directories:
+            source_directories.remove(subvolume_path)
+
+        source_directories.append(snapshot_path)
+        config.setdefault('exclude_patterns', []).append(make_snapshot_exclude_path(subvolume_path))
+
+    return []
+
+
+def delete_snapshot(btrfs_command, snapshot_path):  # pragma: no cover
+    '''
+    Given a Btrfs command to run and the name of a snapshot path, delete it.
+    '''
+    borgmatic.execute.execute_command(
+        (
+            btrfs_command,
+            'subvolume',
+            'delete',
+            snapshot_path,
+        ),
+        output_log_level=logging.DEBUG,
+    )
+
+
+def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run):
+    '''
+    Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime
+    directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use
+    the log prefix in any log entries. If this is a dry run, then don't actually remove anything.
+    '''
+    dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
+
+    btrfs_command = hook_config.get('btrfs_command', 'btrfs')
+    findmnt_command = hook_config.get('findmnt_command', 'findmnt')
+
+    try:
+        all_subvolume_paths = get_subvolumes(btrfs_command, findmnt_command)
+    except FileNotFoundError as error:
+        logger.debug(f'{log_prefix}: Could not find "{error.filename}" command')
+        return
+    except subprocess.CalledProcessError as error:
+        logger.debug(f'{log_prefix}: {error}')
+        return
+
+    for subvolume_path in all_subvolume_paths:
+        subvolume_snapshots_glob = borgmatic.config.paths.replace_temporary_subdirectory_with_glob(
+            os.path.normpath(make_snapshot_path(subvolume_path)),
+            temporary_directory_prefix=BORGMATIC_SNAPSHOT_PREFIX,
+        )
+
+        logger.debug(
+            f'{log_prefix}: Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}'
+        )
+
+        for snapshot_path in glob.glob(subvolume_snapshots_glob):
+            if not os.path.isdir(snapshot_path):
+                continue
+
+            logger.debug(f'{log_prefix}: Deleting Btrfs snapshot {snapshot_path}{dry_run_label}')
+
+            if dry_run:
+                continue
+
+            try:
+                delete_snapshot(btrfs_command, snapshot_path)
+            except FileNotFoundError:
+                logger.debug(f'{log_prefix}: Could not find "{btrfs_command}" command')
+                return
+            except subprocess.CalledProcessError as error:
+                logger.debug(f'{log_prefix}: {error}')
+                return
+
+            # Strip off the subvolume path from the end of the snapshot path and then delete the
+            # resulting directory.
+            shutil.rmtree(snapshot_path.rsplit(subvolume_path, 1)[0])
+
+
+def make_data_source_dump_patterns(
+    hook_config, config, log_prefix, borgmatic_runtime_directory, name=None
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    return ()
+
+
+def restore_data_source_dump(
+    hook_config,
+    config,
+    log_prefix,
+    data_source,
+    dry_run,
+    extract_process,
+    connection_params,
+    borgmatic_runtime_directory,
+):  # pragma: no cover
+    '''
+    Restores aren't implemented, because stored files can be extracted directly with "extract".
+    '''
+    raise NotImplementedError()

borgmatic/hooks/data_source/zfs.py

@@ -104,6 +104,7 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path):  # p
     first).
     '''
     os.makedirs(snapshot_mount_path, mode=0o700, exist_ok=True)
+
     borgmatic.execute.execute_command(
         (
             mount_command,
@@ -131,8 +132,7 @@ def dump_data_sources(
     is a dry run, auto-detect and snapshot any ZFS dataset mount points listed in the given source
     directories and any dataset with a borgmatic-specific user property. Also update those source
     directories, replacing dataset mount points with corresponding snapshot directories so they get
-    stored in the Borg archive instead of the dataset mount points. Use the log prefix in any log
-    entries.
+    stored in the Borg archive instead. Use the log prefix in any log entries.
 
     Return an empty sequence, since there are no ongoing dump processes from this hook.