helpers.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. import argparse
  2. import binascii
  3. import grp
  4. import msgpack
  5. import os
  6. import pwd
  7. import re
  8. import sys
  9. import time
  10. from datetime import datetime, timezone, timedelta
  11. from fnmatch import translate
  12. from operator import attrgetter
  13. import fcntl
  14. import attic.hashindex
  15. import attic.chunker
  16. import attic.crypto
  17. class Error(Exception):
  18. """Error base class"""
  19. exit_code = 1
  20. def get_message(self):
  21. return 'Error: ' + type(self).__doc__.format(*self.args)
  22. class ExtensionModuleError(Error):
  23. """The Attic binary extension modules does not seem to be properly installed"""
  24. class UpgradableLock:
  25. class ReadLockFailed(Error):
  26. """Failed to acquire read lock on {}"""
  27. class WriteLockFailed(Error):
  28. """Failed to acquire write lock on {}"""
  29. def __init__(self, path, exclusive=False):
  30. self.path = path
  31. try:
  32. self.fd = open(path, 'r+')
  33. except IOError:
  34. self.fd = open(path, 'r')
  35. try:
  36. if exclusive:
  37. fcntl.lockf(self.fd, fcntl.LOCK_EX)
  38. else:
  39. fcntl.lockf(self.fd, fcntl.LOCK_SH)
  40. # Python 3.2 raises IOError, Python3.3+ raises OSError
  41. except (IOError, OSError):
  42. if exclusive:
  43. raise self.WriteLockFailed(self.path)
  44. else:
  45. raise self.ReadLockFailed(self.path)
  46. self.is_exclusive = exclusive
  47. def upgrade(self):
  48. try:
  49. fcntl.lockf(self.fd, fcntl.LOCK_EX)
  50. # Python 3.2 raises IOError, Python3.3+ raises OSError
  51. except (IOError, OSError):
  52. raise self.WriteLockFailed(self.path)
  53. self.is_exclusive = True
  54. def release(self):
  55. fcntl.lockf(self.fd, fcntl.LOCK_UN)
  56. self.fd.close()
  57. def check_extension_modules():
  58. import attic.platform
  59. if (attic.hashindex.API_VERSION != 2 or
  60. attic.chunker.API_VERSION != 2 or
  61. attic.crypto.API_VERSION != 2 or
  62. attic.platform.API_VERSION != 2):
  63. raise ExtensionModuleError
  64. class Manifest:
  65. def __init__(self, key, repository):
  66. self.archives = {}
  67. self.config = {}
  68. self.key = key
  69. self.repository = repository
  70. @classmethod
  71. def manifest_id(cls, repository):
  72. return b'\0' * repository.key_size
  73. @classmethod
  74. def load(cls, repository, key=None):
  75. from .key import key_factory
  76. cdata = repository.get(cls.manifest_id(repository))
  77. if not key:
  78. key = key_factory(repository, cdata)
  79. manifest = cls(key, repository)
  80. data = key.decrypt(None, cdata)
  81. manifest.id = key.id_hash(data)
  82. m = msgpack.unpackb(data)
  83. if not m.get(b'version') == 1:
  84. raise ValueError('Invalid manifest version')
  85. manifest.archives = dict((k.decode('utf-8'), v) for k, v in m[b'archives'].items())
  86. manifest.timestamp = m.get(b'timestamp')
  87. if manifest.timestamp:
  88. manifest.timestamp = manifest.timestamp.decode('ascii')
  89. manifest.config = m[b'config']
  90. return manifest, key
  91. def write(self):
  92. self.timestamp = datetime.utcnow().isoformat()
  93. data = msgpack.packb(StableDict({
  94. 'version': 1,
  95. 'archives': self.archives,
  96. 'timestamp': self.timestamp,
  97. 'config': self.config,
  98. }))
  99. self.id = self.key.id_hash(data)
  100. self.repository.put(self.manifest_id(self.repository), self.key.encrypt(data))
  101. def prune_within(archives, within):
  102. multiplier = {'H': 1, 'd': 24, 'w': 24*7, 'm': 24*31, 'y': 24*365}
  103. try:
  104. hours = int(within[:-1]) * multiplier[within[-1]]
  105. except (KeyError, ValueError):
  106. # I don't like how this displays the original exception too:
  107. raise argparse.ArgumentTypeError('Unable to parse --within option: "%s"' % within)
  108. if hours <= 0:
  109. raise argparse.ArgumentTypeError('Number specified using --within option must be positive')
  110. target = datetime.now(timezone.utc) - timedelta(seconds=hours*60*60)
  111. return [a for a in archives if a.ts > target]
  112. def prune_split(archives, pattern, n, skip=[]):
  113. last = None
  114. keep = []
  115. if n == 0:
  116. return keep
  117. for a in sorted(archives, key=attrgetter('ts'), reverse=True):
  118. period = to_localtime(a.ts).strftime(pattern)
  119. if period != last:
  120. last = period
  121. if a not in skip:
  122. keep.append(a)
  123. if len(keep) == n:
  124. break
  125. return keep
  126. class Statistics:
  127. def __init__(self):
  128. self.osize = self.csize = self.usize = self.nfiles = 0
  129. def update(self, size, csize, unique):
  130. self.osize += size
  131. self.csize += csize
  132. if unique:
  133. self.usize += csize
  134. def print_(self, label, cache):
  135. total_size, total_csize, unique_size, unique_csize = cache.chunks.summarize()
  136. print()
  137. print(' Original size Compressed size Deduplicated size')
  138. print('%-15s %20s %20s %20s' % (label, format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize)))
  139. print('All archives: %20s %20s %20s' % (format_file_size(total_size), format_file_size(total_csize), format_file_size(unique_csize)))
  140. def get_keys_dir():
  141. """Determine where to repository keys and cache"""
  142. return os.environ.get('ATTIC_KEYS_DIR',
  143. os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
  144. def get_cache_dir():
  145. """Determine where to repository keys and cache"""
  146. return os.environ.get('ATTIC_CACHE_DIR',
  147. os.path.join(os.path.expanduser('~'), '.cache', 'attic'))
  148. def to_localtime(ts):
  149. """Convert datetime object from UTC to local time zone"""
  150. return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6])
  151. def update_excludes(args):
  152. """Merge exclude patterns from files with those on command line.
  153. Empty lines and lines starting with '#' are ignored, but whitespace
  154. is not stripped."""
  155. if hasattr(args, 'exclude_files') and args.exclude_files:
  156. if not hasattr(args, 'excludes') or args.excludes is None:
  157. args.excludes = []
  158. for file in args.exclude_files:
  159. patterns = [line.rstrip('\r\n') for line in file if not line.startswith('#')]
  160. args.excludes += [ExcludePattern(pattern) for pattern in patterns if pattern]
  161. file.close()
  162. def adjust_patterns(paths, excludes):
  163. if paths:
  164. return (excludes or []) + [IncludePattern(path) for path in paths] + [ExcludePattern('*')]
  165. else:
  166. return excludes
  167. def exclude_path(path, patterns):
  168. """Used by create and extract sub-commands to determine
  169. whether or not an item should be processed.
  170. """
  171. for pattern in (patterns or []):
  172. if pattern.match(path):
  173. return isinstance(pattern, ExcludePattern)
  174. return False
  175. # For both IncludePattern and ExcludePattern, we require that
  176. # the pattern either match the whole path or an initial segment
  177. # of the path up to but not including a path separator. To
  178. # unify the two cases, we add a path separator to the end of
  179. # the path before matching.
  180. class IncludePattern:
  181. """Literal files or directories listed on the command line
  182. for some operations (e.g. extract, but not create).
  183. If a directory is specified, all paths that start with that
  184. path match as well. A trailing slash makes no difference.
  185. """
  186. def __init__(self, pattern):
  187. self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep
  188. def match(self, path):
  189. return (path+os.path.sep).startswith(self.pattern)
  190. def __repr__(self):
  191. return '%s(%s)' % (type(self), self.pattern)
  192. class ExcludePattern(IncludePattern):
  193. """Shell glob patterns to exclude. A trailing slash means to
  194. exclude the contents of a directory, but not the directory itself.
  195. """
  196. def __init__(self, pattern):
  197. if pattern.endswith(os.path.sep):
  198. self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep+'*'+os.path.sep
  199. else:
  200. self.pattern = os.path.normpath(pattern)+os.path.sep+'*'
  201. # fnmatch and re.match both cache compiled regular expressions.
  202. # Nevertheless, this is about 10 times faster.
  203. self.regex = re.compile(translate(self.pattern))
  204. def match(self, path):
  205. return self.regex.match(path+os.path.sep) is not None
  206. def __repr__(self):
  207. return '%s(%s)' % (type(self), self.pattern)
  208. def is_cachedir(path):
  209. """Determines whether the specified path is a cache directory (and
  210. therefore should potentially be excluded from the backup) according to
  211. the CACHEDIR.TAG protocol
  212. (http://www.brynosaurus.com/cachedir/spec.html).
  213. """
  214. tag_contents = b'Signature: 8a477f597d28d172789f06886806bc55'
  215. tag_path = os.path.join(path, 'CACHEDIR.TAG')
  216. try:
  217. if os.path.exists(tag_path):
  218. with open(tag_path, 'rb') as tag_file:
  219. tag_data = tag_file.read(len(tag_contents))
  220. if tag_data == tag_contents:
  221. return True
  222. except OSError:
  223. pass
  224. return False
  225. def format_time(t):
  226. """Format datetime suitable for fixed length list output
  227. """
  228. if abs((datetime.now() - t).days) < 365:
  229. return t.strftime('%b %d %H:%M')
  230. else:
  231. return t.strftime('%b %d %Y')
  232. def format_timedelta(td):
  233. """Format timedelta in a human friendly format
  234. """
  235. # Since td.total_seconds() requires python 2.7
  236. ts = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / float(10 ** 6)
  237. s = ts % 60
  238. m = int(ts / 60) % 60
  239. h = int(ts / 3600) % 24
  240. txt = '%.2f seconds' % s
  241. if m:
  242. txt = '%d minutes %s' % (m, txt)
  243. if h:
  244. txt = '%d hours %s' % (h, txt)
  245. if td.days:
  246. txt = '%d days %s' % (td.days, txt)
  247. return txt
  248. def format_file_mode(mod):
  249. """Format file mode bits for list output
  250. """
  251. def x(v):
  252. return ''.join(v & m and s or '-'
  253. for m, s in ((4, 'r'), (2, 'w'), (1, 'x')))
  254. return '%s%s%s' % (x(mod // 64), x(mod // 8), x(mod))
  255. def format_file_size(v):
  256. """Format file size into a human friendly format
  257. """
  258. if abs(v) > 10**12:
  259. return '%.2f TB' % (v / 10**12)
  260. elif abs(v) > 10**9:
  261. return '%.2f GB' % (v / 10**9)
  262. elif abs(v) > 10**6:
  263. return '%.2f MB' % (v / 10**6)
  264. elif abs(v) > 10**3:
  265. return '%.2f kB' % (v / 10**3)
  266. else:
  267. return '%d B' % v
  268. def format_archive(archive):
  269. return '%-36s %s' % (archive.name, to_localtime(archive.ts).strftime('%c'))
  270. class IntegrityError(Error):
  271. """Data integrity error"""
  272. def memoize(function):
  273. cache = {}
  274. def decorated_function(*args):
  275. try:
  276. return cache[args]
  277. except KeyError:
  278. val = function(*args)
  279. cache[args] = val
  280. return val
  281. return decorated_function
  282. @memoize
  283. def uid2user(uid, default=None):
  284. try:
  285. return pwd.getpwuid(uid).pw_name
  286. except KeyError:
  287. return default
  288. @memoize
  289. def user2uid(user, default=None):
  290. try:
  291. return user and pwd.getpwnam(user).pw_uid
  292. except KeyError:
  293. return default
  294. @memoize
  295. def gid2group(gid, default=None):
  296. try:
  297. return grp.getgrgid(gid).gr_name
  298. except KeyError:
  299. return default
  300. @memoize
  301. def group2gid(group, default=None):
  302. try:
  303. return group and grp.getgrnam(group).gr_gid
  304. except KeyError:
  305. return default
  306. def posix_acl_use_stored_uid_gid(acl):
  307. """Replace the user/group field with the stored uid/gid
  308. """
  309. entries = []
  310. for entry in acl.decode('ascii').split('\n'):
  311. if entry:
  312. fields = entry.split(':')
  313. if len(fields) == 4:
  314. entries.append(':'.join([fields[0], fields[3], fields[2]]))
  315. else:
  316. entries.append(entry)
  317. return ('\n'.join(entries)).encode('ascii')
  318. class Location:
  319. """Object representing a repository / archive location
  320. """
  321. proto = user = host = port = path = archive = None
  322. ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
  323. r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
  324. r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
  325. file_re = re.compile(r'(?P<proto>file)://'
  326. r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
  327. scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
  328. r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
  329. def __init__(self, text):
  330. self.orig = text
  331. if not self.parse(text):
  332. raise ValueError
  333. def parse(self, text):
  334. m = self.ssh_re.match(text)
  335. if m:
  336. self.proto = m.group('proto')
  337. self.user = m.group('user')
  338. self.host = m.group('host')
  339. self.port = m.group('port') and int(m.group('port')) or None
  340. self.path = m.group('path')
  341. self.archive = m.group('archive')
  342. return True
  343. m = self.file_re.match(text)
  344. if m:
  345. self.proto = m.group('proto')
  346. self.path = m.group('path')
  347. self.archive = m.group('archive')
  348. return True
  349. m = self.scp_re.match(text)
  350. if m:
  351. self.user = m.group('user')
  352. self.host = m.group('host')
  353. self.path = m.group('path')
  354. self.archive = m.group('archive')
  355. self.proto = self.host and 'ssh' or 'file'
  356. return True
  357. return False
  358. def __str__(self):
  359. items = []
  360. items.append('proto=%r' % self.proto)
  361. items.append('user=%r' % self.user)
  362. items.append('host=%r' % self.host)
  363. items.append('port=%r' % self.port)
  364. items.append('path=%r' % self.path)
  365. items.append('archive=%r' % self.archive)
  366. return ', '.join(items)
  367. def to_key_filename(self):
  368. name = re.sub('[^\w]', '_', self.path).strip('_')
  369. if self.proto != 'file':
  370. name = self.host + '__' + name
  371. return os.path.join(get_keys_dir(), name)
  372. def __repr__(self):
  373. return "Location(%s)" % self
  374. def location_validator(archive=None):
  375. def validator(text):
  376. try:
  377. loc = Location(text)
  378. except ValueError:
  379. raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text)
  380. if archive is True and not loc.archive:
  381. raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
  382. elif archive is False and loc.archive:
  383. raise argparse.ArgumentTypeError('"%s" No archive can be specified' % text)
  384. return loc
  385. return validator
  386. def read_msgpack(filename):
  387. with open(filename, 'rb') as fd:
  388. return msgpack.unpack(fd)
  389. def write_msgpack(filename, d):
  390. with open(filename + '.tmp', 'wb') as fd:
  391. msgpack.pack(d, fd)
  392. fd.flush()
  393. os.fsync(fd.fileno())
  394. os.rename(filename + '.tmp', filename)
  395. def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'):
  396. for key in keys:
  397. if isinstance(d.get(key), bytes):
  398. d[key] = d[key].decode(encoding, errors)
  399. return d
  400. def remove_surrogates(s, errors='replace'):
  401. """Replace surrogates generated by fsdecode with '?'
  402. """
  403. return s.encode('utf-8', errors).decode('utf-8')
  404. _safe_re = re.compile('^((..)?/+)+')
  405. def make_path_safe(path):
  406. """Make path safe by making it relative and local
  407. """
  408. return _safe_re.sub('', path) or '.'
  409. def daemonize():
  410. """Detach process from controlling terminal and run in background
  411. """
  412. pid = os.fork()
  413. if pid:
  414. os._exit(0)
  415. os.setsid()
  416. pid = os.fork()
  417. if pid:
  418. os._exit(0)
  419. os.chdir('/')
  420. os.close(0)
  421. os.close(1)
  422. os.close(2)
  423. fd = os.open('/dev/null', os.O_RDWR)
  424. os.dup2(fd, 0)
  425. os.dup2(fd, 1)
  426. os.dup2(fd, 2)
  427. class StableDict(dict):
  428. """A dict subclass with stable items() ordering"""
  429. def items(self):
  430. return sorted(super(StableDict, self).items())
  431. if sys.version < '3.3':
  432. # st_mtime_ns attribute only available in 3.3+
  433. def st_mtime_ns(st):
  434. return int(st.st_mtime * 1e9)
  435. # unhexlify in < 3.3 incorrectly only accepts bytes input
  436. def unhexlify(data):
  437. if isinstance(data, str):
  438. data = data.encode('ascii')
  439. return binascii.unhexlify(data)
  440. else:
  441. def st_mtime_ns(st):
  442. return st.st_mtime_ns
  443. unhexlify = binascii.unhexlify
  444. def bigint_to_int(mtime):
  445. """Convert bytearray to int
  446. """
  447. if isinstance(mtime, bytes):
  448. return int.from_bytes(mtime, 'little', signed=True)
  449. return mtime
  450. def int_to_bigint(value):
  451. """Convert integers larger than 64 bits to bytearray
  452. Smaller integers are left alone
  453. """
  454. if value.bit_length() > 63:
  455. return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True)
  456. return value