|  | @@ -1,4 +1,3 @@
 | 
	
		
			
				|  |  | -import glob
 | 
	
		
			
				|  |  |  import itertools
 | 
	
		
			
				|  |  |  import logging
 | 
	
		
			
				|  |  |  import os
 | 
	
	
		
			
				|  | @@ -20,31 +19,6 @@ from borgmatic.execute import (
 | 
	
		
			
				|  |  |  logger = logging.getLogger(__name__)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -def expand_directory(directory, working_directory):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Given a directory path, expand any tilde (representing a user's home directory) and any globs
 | 
	
		
			
				|  |  | -    therein. Return a list of one or more resulting paths.
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    expanded_directory = os.path.join(working_directory or '', os.path.expanduser(directory))
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    return glob.glob(expanded_directory) or [expanded_directory]
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -def expand_directories(directories, working_directory=None):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Given a sequence of directory paths and an optional working directory, expand tildes and globs
 | 
	
		
			
				|  |  | -    in each one. Return all the resulting directories as a single flattened tuple.
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    if directories is None:
 | 
	
		
			
				|  |  | -        return ()
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    return tuple(
 | 
	
		
			
				|  |  | -        itertools.chain.from_iterable(
 | 
	
		
			
				|  |  | -            expand_directory(directory, working_directory) for directory in directories
 | 
	
		
			
				|  |  | -        )
 | 
	
		
			
				|  |  | -    )
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |  def expand_home_directories(directories):
 | 
	
		
			
				|  |  |      '''
 | 
	
		
			
				|  |  |      Given a sequence of directory paths, expand tildes in each one. Do not perform any globbing.
 | 
	
	
		
			
				|  | @@ -56,67 +30,6 @@ def expand_home_directories(directories):
 | 
	
		
			
				|  |  |      return tuple(os.path.expanduser(directory) for directory in directories)
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -def map_directories_to_devices(directories, working_directory=None):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Given a sequence of directories and an optional working directory, return a map from directory
 | 
	
		
			
				|  |  | -    to an identifier for the device on which that directory resides or None if the path doesn't
 | 
	
		
			
				|  |  | -    exist.
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    This is handy for determining whether two different directories are on the same filesystem (have
 | 
	
		
			
				|  |  | -    the same device identifier).
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    return {
 | 
	
		
			
				|  |  | -        directory: os.stat(full_directory).st_dev if os.path.exists(full_directory) else None
 | 
	
		
			
				|  |  | -        for directory in directories
 | 
	
		
			
				|  |  | -        for full_directory in (os.path.join(working_directory or '', directory),)
 | 
	
		
			
				|  |  | -    }
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -def deduplicate_directories(directory_devices, additional_directory_devices):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Given a map from directory to the identifier for the device on which that directory resides,
 | 
	
		
			
				|  |  | -    return the directories as a sorted tuple with all duplicate child directories removed. For
 | 
	
		
			
				|  |  | -    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    The one exception to this rule is if two paths are on different filesystems (devices). In that
 | 
	
		
			
				|  |  | -    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
 | 
	
		
			
				|  |  | -    location.one_file_system option is true).
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
 | 
	
		
			
				|  |  | -    child directories, because it will naturally spider the contents of the parent directory. And
 | 
	
		
			
				|  |  | -    there are cases where Borg coming across the same file twice will result in duplicate reads and
 | 
	
		
			
				|  |  | -    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
 | 
	
		
			
				|  |  | -    Borg.
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    If any additional directory devices are given, also deduplicate against them, but don't include
 | 
	
		
			
				|  |  | -    them in the returned directories.
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    deduplicated = set()
 | 
	
		
			
				|  |  | -    directories = sorted(directory_devices.keys())
 | 
	
		
			
				|  |  | -    additional_directories = sorted(additional_directory_devices.keys())
 | 
	
		
			
				|  |  | -    all_devices = {**directory_devices, **additional_directory_devices}
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    for directory in directories:
 | 
	
		
			
				|  |  | -        deduplicated.add(directory)
 | 
	
		
			
				|  |  | -        parents = pathlib.PurePath(directory).parents
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -        # If another directory in the given list (or the additional list) is a parent of current
 | 
	
		
			
				|  |  | -        # directory (even n levels up) and both are on the same filesystem, then the current
 | 
	
		
			
				|  |  | -        # directory is a duplicate.
 | 
	
		
			
				|  |  | -        for other_directory in directories + additional_directories:
 | 
	
		
			
				|  |  | -            for parent in parents:
 | 
	
		
			
				|  |  | -                if (
 | 
	
		
			
				|  |  | -                    pathlib.PurePath(other_directory) == parent
 | 
	
		
			
				|  |  | -                    and all_devices[directory] is not None
 | 
	
		
			
				|  |  | -                    and all_devices[other_directory] == all_devices[directory]
 | 
	
		
			
				|  |  | -                ):
 | 
	
		
			
				|  |  | -                    if directory in deduplicated:
 | 
	
		
			
				|  |  | -                        deduplicated.remove(directory)
 | 
	
		
			
				|  |  | -                    break
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    return tuple(sorted(deduplicated))
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |  def write_pattern_file(patterns=None, sources=None, pattern_file=None):
 | 
	
		
			
				|  |  |      '''
 | 
	
		
			
				|  |  |      Given a sequence of patterns and an optional sequence of source directories, write them to a
 | 
	
	
		
			
				|  | @@ -221,32 +134,9 @@ def make_list_filter_flags(local_borg_version, dry_run):
 | 
	
		
			
				|  |  |          return f'{base_flags}-'
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -def collect_borgmatic_runtime_directories(borgmatic_runtime_directory):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Return a list of borgmatic-specific runtime directories used for temporary runtime data like
 | 
	
		
			
				|  |  | -    streaming database dumps and bootstrap metadata. If no such directories exist, return an empty
 | 
	
		
			
				|  |  | -    list.
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    return [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |  ROOT_PATTERN_PREFIX = 'R '
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -def pattern_root_directories(patterns=None):
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    Given a sequence of patterns, parse out and return just the root directories.
 | 
	
		
			
				|  |  | -    '''
 | 
	
		
			
				|  |  | -    if not patterns:
 | 
	
		
			
				|  |  | -        return []
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -    return [
 | 
	
		
			
				|  |  | -        pattern.split(ROOT_PATTERN_PREFIX, maxsplit=1)[1]
 | 
	
		
			
				|  |  | -        for pattern in patterns
 | 
	
		
			
				|  |  | -        if pattern.startswith(ROOT_PATTERN_PREFIX)
 | 
	
		
			
				|  |  | -    ]
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |  def special_file(path):
 | 
	
		
			
				|  |  |      '''
 | 
	
		
			
				|  |  |      Return whether the given path is a special file (character device, block device, or named pipe
 | 
	
	
		
			
				|  | @@ -335,9 +225,10 @@ def make_base_create_command(
 | 
	
		
			
				|  |  |      repository_path,
 | 
	
		
			
				|  |  |      config,
 | 
	
		
			
				|  |  |      config_paths,
 | 
	
		
			
				|  |  | +    source_directories,
 | 
	
		
			
				|  |  |      local_borg_version,
 | 
	
		
			
				|  |  |      global_arguments,
 | 
	
		
			
				|  |  | -    borgmatic_runtime_directories,
 | 
	
		
			
				|  |  | +    borgmatic_runtime_directory,
 | 
	
		
			
				|  |  |      local_path='borg',
 | 
	
		
			
				|  |  |      remote_path=None,
 | 
	
		
			
				|  |  |      progress=False,
 | 
	
	
		
			
				|  | @@ -359,27 +250,10 @@ def make_base_create_command(
 | 
	
		
			
				|  |  |              config.get('source_directories'), working_directory=working_directory
 | 
	
		
			
				|  |  |          )
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  | -    sources = deduplicate_directories(
 | 
	
		
			
				|  |  | -        map_directories_to_devices(
 | 
	
		
			
				|  |  | -            expand_directories(
 | 
	
		
			
				|  |  | -                tuple(config.get('source_directories', ()))
 | 
	
		
			
				|  |  | -                + borgmatic_runtime_directories
 | 
	
		
			
				|  |  | -                + tuple(config_paths if config.get('store_config_files', True) else ()),
 | 
	
		
			
				|  |  | -                working_directory=working_directory,
 | 
	
		
			
				|  |  | -            )
 | 
	
		
			
				|  |  | -        ),
 | 
	
		
			
				|  |  | -        additional_directory_devices=map_directories_to_devices(
 | 
	
		
			
				|  |  | -            expand_directories(
 | 
	
		
			
				|  |  | -                pattern_root_directories(config.get('patterns')),
 | 
	
		
			
				|  |  | -                working_directory=working_directory,
 | 
	
		
			
				|  |  | -            )
 | 
	
		
			
				|  |  | -        ),
 | 
	
		
			
				|  |  | -    )
 | 
	
		
			
				|  |  | -
 | 
	
		
			
				|  |  |      ensure_files_readable(config.get('patterns_from'), config.get('exclude_from'))
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      pattern_file = (
 | 
	
		
			
				|  |  | -        write_pattern_file(config.get('patterns'), sources)
 | 
	
		
			
				|  |  | +        write_pattern_file(config.get('patterns'), source_directories)
 | 
	
		
			
				|  |  |          if config.get('patterns') or config.get('patterns_from')
 | 
	
		
			
				|  |  |          else None
 | 
	
		
			
				|  |  |      )
 | 
	
	
		
			
				|  | @@ -457,7 +331,7 @@ def make_base_create_command(
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      create_positional_arguments = flags.make_repository_archive_flags(
 | 
	
		
			
				|  |  |          repository_path, archive_name_format, local_borg_version
 | 
	
		
			
				|  |  | -    ) + (sources if not pattern_file else ())
 | 
	
		
			
				|  |  | +    ) + (tuple(source_directories) if not pattern_file else ())
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      # If database hooks are enabled (as indicated by streaming processes), exclude files that might
 | 
	
		
			
				|  |  |      # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
 | 
	
	
		
			
				|  | @@ -474,7 +348,9 @@ def make_base_create_command(
 | 
	
		
			
				|  |  |              local_path,
 | 
	
		
			
				|  |  |              working_directory,
 | 
	
		
			
				|  |  |              borg_environment,
 | 
	
		
			
				|  |  | -            skip_directories=borgmatic_runtime_directories,
 | 
	
		
			
				|  |  | +            skip_directories=(
 | 
	
		
			
				|  |  | +                [borgmatic_runtime_directory] if os.path.exists(borgmatic_runtime_directory) else []
 | 
	
		
			
				|  |  | +            ),
 | 
	
		
			
				|  |  |          )
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |          if special_file_paths:
 | 
	
	
		
			
				|  | @@ -502,6 +378,7 @@ def create_archive(
 | 
	
		
			
				|  |  |      repository_path,
 | 
	
		
			
				|  |  |      config,
 | 
	
		
			
				|  |  |      config_paths,
 | 
	
		
			
				|  |  | +    source_directories,
 | 
	
		
			
				|  |  |      local_borg_version,
 | 
	
		
			
				|  |  |      global_arguments,
 | 
	
		
			
				|  |  |      borgmatic_runtime_directory,
 | 
	
	
		
			
				|  | @@ -524,10 +401,6 @@ def create_archive(
 | 
	
		
			
				|  |  |      borgmatic.logger.add_custom_log_levels()
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      working_directory = borgmatic.config.paths.get_working_directory(config)
 | 
	
		
			
				|  |  | -    borgmatic_runtime_directories = expand_directories(
 | 
	
		
			
				|  |  | -        collect_borgmatic_runtime_directories(borgmatic_runtime_directory),
 | 
	
		
			
				|  |  | -        working_directory=working_directory,
 | 
	
		
			
				|  |  | -    )
 | 
	
		
			
				|  |  |  
 | 
	
		
			
				|  |  |      (create_flags, create_positional_arguments, pattern_file, exclude_file) = (
 | 
	
		
			
				|  |  |          make_base_create_command(
 | 
	
	
		
			
				|  | @@ -535,9 +408,10 @@ def create_archive(
 | 
	
		
			
				|  |  |              repository_path,
 | 
	
		
			
				|  |  |              config,
 | 
	
		
			
				|  |  |              config_paths,
 | 
	
		
			
				|  |  | +            source_directories,
 | 
	
		
			
				|  |  |              local_borg_version,
 | 
	
		
			
				|  |  |              global_arguments,
 | 
	
		
			
				|  |  | -            borgmatic_runtime_directories,
 | 
	
		
			
				|  |  | +            borgmatic_runtime_directory,
 | 
	
		
			
				|  |  |              local_path,
 | 
	
		
			
				|  |  |              remote_path,
 | 
	
		
			
				|  |  |              progress,
 |