shared.py 5.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. from datetime import datetime
  2. import os
  3. import platform
  4. import subprocess
  5. from atticmatic.config import Section_format, option
  6. from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
  7. # Common backend functionality shared by Attic and Borg. As the two backup
  8. # commands diverge, these shared functions will likely need to be replaced
  9. # with non-shared functions within atticmatic.backends.attic and
  10. # atticmatic.backends.borg.
  11. CONFIG_FORMAT = (
  12. Section_format(
  13. 'location',
  14. (
  15. option('source_directories'),
  16. option('repository'),
  17. ),
  18. ),
  19. Section_format(
  20. 'retention',
  21. (
  22. option('keep_within', required=False),
  23. option('keep_hourly', int, required=False),
  24. option('keep_daily', int, required=False),
  25. option('keep_weekly', int, required=False),
  26. option('keep_monthly', int, required=False),
  27. option('keep_yearly', int, required=False),
  28. option('prefix', required=False),
  29. ),
  30. ),
  31. Section_format(
  32. 'consistency',
  33. (
  34. option('checks', required=False),
  35. ),
  36. )
  37. )
  38. def create_archive(excludes_filename, verbosity, source_directories, repository, command):
  39. '''
  40. Given an excludes filename (or None), a vebosity flag, a space-separated list of source
  41. directories, a local or remote repository path, and a command to run, create an attic archive.
  42. '''
  43. sources = tuple(source_directories.split(' '))
  44. exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
  45. verbosity_flags = {
  46. VERBOSITY_SOME: ('--stats',),
  47. VERBOSITY_LOTS: ('--verbose', '--stats'),
  48. }.get(verbosity, ())
  49. full_command = (
  50. command, 'create',
  51. '{repo}::{hostname}-{timestamp}'.format(
  52. repo=repository,
  53. hostname=platform.node(),
  54. timestamp=datetime.now().isoformat(),
  55. ),
  56. ) + sources + exclude_flags + verbosity_flags
  57. subprocess.check_call(full_command)
  58. def _make_prune_flags(retention_config):
  59. '''
  60. Given a retention config dict mapping from option name to value, tranform it into an iterable of
  61. command-line name-value flag pairs.
  62. For example, given a retention config of:
  63. {'keep_weekly': 4, 'keep_monthly': 6}
  64. This will be returned as an iterable of:
  65. (
  66. ('--keep-weekly', '4'),
  67. ('--keep-monthly', '6'),
  68. )
  69. '''
  70. return (
  71. ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
  72. for option_name, value in retention_config.items()
  73. )
  74. def prune_archives(verbosity, repository, retention_config, command):
  75. '''
  76. Given a verbosity flag, a local or remote repository path, a retention config dict, and a
  77. command to run, prune attic archives according the the retention policy specified in that
  78. configuration.
  79. '''
  80. verbosity_flags = {
  81. VERBOSITY_SOME: ('--stats',),
  82. VERBOSITY_LOTS: ('--verbose', '--stats'),
  83. }.get(verbosity, ())
  84. full_command = (
  85. command, 'prune',
  86. repository,
  87. ) + tuple(
  88. element
  89. for pair in _make_prune_flags(retention_config)
  90. for element in pair
  91. ) + verbosity_flags
  92. subprocess.check_call(full_command)
  93. DEFAULT_CHECKS = ('repository', 'archives')
  94. def _parse_checks(consistency_config):
  95. '''
  96. Given a consistency config with a space-separated "checks" option, transform it to a tuple of
  97. named checks to run.
  98. For example, given a retention config of:
  99. {'checks': 'repository archives'}
  100. This will be returned as:
  101. ('repository', 'archives')
  102. If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
  103. "disabled", return an empty tuple, meaning that no checks should be run.
  104. '''
  105. checks = consistency_config.get('checks', '').strip()
  106. if not checks:
  107. return DEFAULT_CHECKS
  108. return tuple(
  109. check for check in consistency_config['checks'].split(' ')
  110. if check.lower() not in ('disabled', '')
  111. )
  112. def _make_check_flags(checks, check_last=None):
  113. '''
  114. Given a parsed sequence of checks, transform it into tuple of command-line flags.
  115. For example, given parsed checks of:
  116. ('repository',)
  117. This will be returned as:
  118. ('--repository-only',)
  119. Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
  120. Borg supports this flag.
  121. '''
  122. last_flag = ('--last', check_last) if check_last else ()
  123. if checks == DEFAULT_CHECKS:
  124. return last_flag
  125. return tuple(
  126. '--{}-only'.format(check) for check in checks
  127. ) + last_flag
  128. def check_archives(verbosity, repository, consistency_config, command):
  129. '''
  130. Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
  131. command to run, check the contained attic archives for consistency.
  132. If there are no consistency checks to run, skip running them.
  133. '''
  134. checks = _parse_checks(consistency_config)
  135. check_last = consistency_config.get('check_last', None)
  136. if not checks:
  137. return
  138. verbosity_flags = {
  139. VERBOSITY_SOME: ('--verbose',),
  140. VERBOSITY_LOTS: ('--verbose',),
  141. }.get(verbosity, ())
  142. full_command = (
  143. command, 'check',
  144. repository,
  145. ) + _make_check_flags(checks, check_last) + verbosity_flags
  146. # The check command spews to stdout even without the verbose flag. Suppress it.
  147. stdout = None if verbosity_flags else open(os.devnull, 'w')
  148. subprocess.check_call(full_command, stdout=stdout)