|
@@ -1,4 +1,3 @@
|
|
|
-import glob
|
|
|
import itertools
|
|
|
import logging
|
|
|
import os
|
|
@@ -20,31 +19,6 @@ from borgmatic.execute import (
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
-def expand_directory(directory, working_directory):
|
|
|
- '''
|
|
|
- Given a directory path, expand any tilde (representing a user's home directory) and any globs
|
|
|
- therein. Return a list of one or more resulting paths.
|
|
|
- '''
|
|
|
- expanded_directory = os.path.join(working_directory or '', os.path.expanduser(directory))
|
|
|
-
|
|
|
- return glob.glob(expanded_directory) or [expanded_directory]
|
|
|
-
|
|
|
-
|
|
|
-def expand_directories(directories, working_directory=None):
|
|
|
- '''
|
|
|
- Given a sequence of directory paths and an optional working directory, expand tildes and globs
|
|
|
- in each one. Return all the resulting directories as a single flattened tuple.
|
|
|
- '''
|
|
|
- if directories is None:
|
|
|
- return ()
|
|
|
-
|
|
|
- return tuple(
|
|
|
- itertools.chain.from_iterable(
|
|
|
- expand_directory(directory, working_directory) for directory in directories
|
|
|
- )
|
|
|
- )
|
|
|
-
|
|
|
-
|
|
|
def expand_home_directories(directories):
|
|
|
'''
|
|
|
Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
|
|
@@ -56,67 +30,6 @@ def expand_home_directories(directories):
|
|
|
return tuple(os.path.expanduser(directory) for directory in directories)
|
|
|
|
|
|
|
|
|
-def map_directories_to_devices(directories, working_directory=None):
|
|
|
- '''
|
|
|
- Given a sequence of directories and an optional working directory, return a map from directory
|
|
|
- to an identifier for the device on which that directory resides or None if the path doesn't
|
|
|
- exist.
|
|
|
-
|
|
|
- This is handy for determining whether two different directories are on the same filesystem (have
|
|
|
- the same device identifier).
|
|
|
- '''
|
|
|
- return {
|
|
|
- directory: os.stat(full_directory).st_dev if os.path.exists(full_directory) else None
|
|
|
- for directory in directories
|
|
|
- for full_directory in (os.path.join(working_directory or '', directory),)
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-def deduplicate_directories(directory_devices, additional_directory_devices):
|
|
|
- '''
|
|
|
- Given a map from directory to the identifier for the device on which that directory resides,
|
|
|
- return the directories as a sorted tuple with all duplicate child directories removed. For
|
|
|
- instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
|
|
|
-
|
|
|
- The one exception to this rule is if two paths are on different filesystems (devices). In that
|
|
|
- case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
|
|
|
- location.one_file_system option is true).
|
|
|
-
|
|
|
- The idea is that if Borg is given a parent directory, then it doesn't also need to be given
|
|
|
- child directories, because it will naturally spider the contents of the parent directory. And
|
|
|
- there are cases where Borg coming across the same file twice will result in duplicate reads and
|
|
|
- even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
|
|
|
- Borg.
|
|
|
-
|
|
|
- If any additional directory devices are given, also deduplicate against them, but don't include
|
|
|
- them in the returned directories.
|
|
|
- '''
|
|
|
- deduplicated = set()
|
|
|
- directories = sorted(directory_devices.keys())
|
|
|
- additional_directories = sorted(additional_directory_devices.keys())
|
|
|
- all_devices = {**directory_devices, **additional_directory_devices}
|
|
|
-
|
|
|
- for directory in directories:
|
|
|
- deduplicated.add(directory)
|
|
|
- parents = pathlib.PurePath(directory).parents
|
|
|
-
|
|
|
- # If another directory in the given list (or the additional list) is a parent of current
|
|
|
- # directory (even n levels up) and both are on the same filesystem, then the current
|
|
|
- # directory is a duplicate.
|
|
|
- for other_directory in directories + additional_directories:
|
|
|
- for parent in parents:
|
|
|
- if (
|
|
|
- pathlib.PurePath(other_directory) == parent
|
|
|
- and all_devices[directory] is not None
|
|
|
- and all_devices[other_directory] == all_devices[directory]
|
|
|
- ):
|
|
|
- if directory in deduplicated:
|
|
|
- deduplicated.remove(directory)
|
|
|
- break
|
|
|
-
|
|
|
- return tuple(sorted(deduplicated))
|
|
|
-
|
|
|
-
|
|
|
def write_pattern_file(patterns=None, sources=None, pattern_file=None):
|
|
|
'''
|
|
|
Given a sequence of patterns and an optional sequence of source directories, write them to a
|
|
@@ -221,32 +134,9 @@ def make_list_filter_flags(local_borg_version, dry_run):
|
|
|
return f'{base_flags}-'
|
|
|
|
|
|
|
|
|
-def collect_borgmatic_runtime_directories(borgmatic_runtime_directory):
|
|
|
- '''
|
|
|
- Return a list of borgmatic-specific runtime directories used for temporary runtime data like
|
|
|
- streaming database dumps and bootstrap metadata. If no such directories exist, return an empty
|
|
|
- list.
|
|
|
- '''
|
|
|
- return [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
|
|
|
-
|
|
|
-
|
|
|
ROOT_PATTERN_PREFIX = 'R '
|
|
|
|
|
|
|
|
|
-def pattern_root_directories(patterns=None):
|
|
|
- '''
|
|
|
- Given a sequence of patterns, parse out and return just the root directories.
|
|
|
- '''
|
|
|
- if not patterns:
|
|
|
- return []
|
|
|
-
|
|
|
- return [
|
|
|
- pattern.split(ROOT_PATTERN_PREFIX, maxsplit=1)[1]
|
|
|
- for pattern in patterns
|
|
|
- if pattern.startswith(ROOT_PATTERN_PREFIX)
|
|
|
- ]
|
|
|
-
|
|
|
-
|
|
|
def special_file(path):
|
|
|
'''
|
|
|
Return whether the given path is a special file (character device, block device, or named pipe
|
|
@@ -335,9 +225,10 @@ def make_base_create_command(
|
|
|
repository_path,
|
|
|
config,
|
|
|
config_paths,
|
|
|
+ source_directories,
|
|
|
local_borg_version,
|
|
|
global_arguments,
|
|
|
- borgmatic_runtime_directories,
|
|
|
+ borgmatic_runtime_directory,
|
|
|
local_path='borg',
|
|
|
remote_path=None,
|
|
|
progress=False,
|
|
@@ -359,27 +250,10 @@ def make_base_create_command(
|
|
|
config.get('source_directories'), working_directory=working_directory
|
|
|
)
|
|
|
|
|
|
- sources = deduplicate_directories(
|
|
|
- map_directories_to_devices(
|
|
|
- expand_directories(
|
|
|
- tuple(config.get('source_directories', ()))
|
|
|
- + borgmatic_runtime_directories
|
|
|
- + tuple(config_paths if config.get('store_config_files', True) else ()),
|
|
|
- working_directory=working_directory,
|
|
|
- )
|
|
|
- ),
|
|
|
- additional_directory_devices=map_directories_to_devices(
|
|
|
- expand_directories(
|
|
|
- pattern_root_directories(config.get('patterns')),
|
|
|
- working_directory=working_directory,
|
|
|
- )
|
|
|
- ),
|
|
|
- )
|
|
|
-
|
|
|
ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
|
|
|
|
|
|
pattern_file = (
|
|
|
- write_pattern_file(config.get('patterns'), sources)
|
|
|
+ write_pattern_file(config.get('patterns'), source_directories)
|
|
|
if config.get('patterns') or config.get('patterns_from')
|
|
|
else None
|
|
|
)
|
|
@@ -457,7 +331,7 @@ def make_base_create_command(
|
|
|
|
|
|
create_positional_arguments = flags.make_repository_archive_flags(
|
|
|
repository_path, archive_name_format, local_borg_version
|
|
|
- ) + (sources if not pattern_file else ())
|
|
|
+ ) + (tuple(source_directories) if not pattern_file else ())
|
|
|
|
|
|
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
|
|
|
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
|
|
@@ -474,7 +348,9 @@ def make_base_create_command(
|
|
|
local_path,
|
|
|
working_directory,
|
|
|
borg_environment,
|
|
|
- skip_directories=borgmatic_runtime_directories,
|
|
|
+ skip_directories=(
|
|
|
+ [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
|
|
|
+ ),
|
|
|
)
|
|
|
|
|
|
if special_file_paths:
|
|
@@ -502,6 +378,7 @@ def create_archive(
|
|
|
repository_path,
|
|
|
config,
|
|
|
config_paths,
|
|
|
+ source_directories,
|
|
|
local_borg_version,
|
|
|
global_arguments,
|
|
|
borgmatic_runtime_directory,
|
|
@@ -524,10 +401,6 @@ def create_archive(
|
|
|
borgmatic.logger.add_custom_log_levels()
|
|
|
|
|
|
working_directory = borgmatic.config.paths.get_working_directory(config)
|
|
|
- borgmatic_runtime_directories = expand_directories(
|
|
|
- collect_borgmatic_runtime_directories(borgmatic_runtime_directory),
|
|
|
- working_directory=working_directory,
|
|
|
- )
|
|
|
|
|
|
(create_flags, create_positional_arguments, pattern_file, exclude_file) = (
|
|
|
make_base_create_command(
|
|
@@ -535,9 +408,10 @@ def create_archive(
|
|
|
repository_path,
|
|
|
config,
|
|
|
config_paths,
|
|
|
+ source_directories,
|
|
|
local_borg_version,
|
|
|
global_arguments,
|
|
|
- borgmatic_runtime_directories,
|
|
|
+ borgmatic_runtime_directory,
|
|
|
local_path,
|
|
|
remote_path,
|
|
|
progress,
|