upgrader.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. from binascii import hexlify
  2. import logging
  3. logger = logging.getLogger(__name__)
  4. import os
  5. import shutil
  6. import sys
  7. import time
  8. from .helpers import get_keys_dir, get_cache_dir
  9. from .locking import UpgradableLock
  10. from .repository import Repository, MAGIC
  11. from .key import KeyfileKey, KeyfileNotFoundError
  12. ATTIC_MAGIC = b'ATTICSEG'
  13. class AtticRepositoryUpgrader(Repository):
  14. def upgrade(self, dryrun=True):
  15. """convert an attic repository to a borg repository
  16. those are the files that need to be upgraded here, from most
  17. important to least important: segments, key files, and various
  18. caches, the latter being optional, as they will be rebuilt if
  19. missing.
  20. we nevertheless do the order in reverse, as we prefer to do
  21. the fast stuff first, to improve interactivity.
  22. """
  23. logger.info("opening attic repository with borg and converting")
  24. # we need to open the repo to load configuration, keyfiles and segments
  25. self.open(self.path, exclusive=False)
  26. segments = [filename for i, filename in self.io.segment_iterator()]
  27. try:
  28. keyfile = self.find_attic_keyfile()
  29. except KeyfileNotFoundError:
  30. logger.warning("no key file found for repository")
  31. else:
  32. self.convert_keyfiles(keyfile, dryrun)
  33. self.close()
  34. # partial open: just hold on to the lock
  35. self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
  36. exclusive=True).acquire()
  37. try:
  38. self.convert_cache(dryrun)
  39. self.convert_segments(segments, dryrun)
  40. finally:
  41. self.lock.release()
  42. self.lock = None
  43. @staticmethod
  44. def convert_segments(segments, dryrun):
  45. """convert repository segments from attic to borg
  46. replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
  47. `$ATTIC_REPO/data/**`.
  48. luckily the magic string length didn't change so we can just
  49. replace the 8 first bytes of all regular files in there."""
  50. logger.info("converting %d segments..." % len(segments))
  51. i = 0
  52. for filename in segments:
  53. i += 1
  54. print("\rconverting segment %d/%d in place, %.2f%% done (%s)"
  55. % (i, len(segments), 100*float(i)/len(segments), filename),
  56. end='', file=sys.stderr)
  57. if dryrun:
  58. time.sleep(0.001)
  59. else:
  60. AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC)
  61. print(file=sys.stderr)
  62. @staticmethod
  63. def header_replace(filename, old_magic, new_magic):
  64. with open(filename, 'r+b') as segment:
  65. segment.seek(0)
  66. # only write if necessary
  67. if segment.read(len(old_magic)) == old_magic:
  68. segment.seek(0)
  69. segment.write(new_magic)
  70. def find_attic_keyfile(self):
  71. """find the attic keyfiles
  72. the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
  73. finds the keys with the right identifier for the repo.
  74. this is expected to look into $HOME/.attic/keys or
  75. $ATTIC_KEYS_DIR for key files matching the given Borg
  76. repository.
  77. it is expected to raise an exception (KeyfileNotFoundError) if
  78. no key is found. whether that exception is from Borg or Attic
  79. is unclear.
  80. this is split in a separate function in case we want to use
  81. the attic code here directly, instead of our local
  82. implementation."""
  83. return AtticKeyfileKey.find_key_file(self)
  84. @staticmethod
  85. def convert_keyfiles(keyfile, dryrun):
  86. """convert key files from attic to borg
  87. replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
  88. `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
  89. `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
  90. `$HOME/.borg/keys`.
  91. no need to decrypt to convert. we need to rewrite the whole
  92. key file because magic string length changed, but that's not a
  93. problem because the keyfiles are small (compared to, say,
  94. all the segments)."""
  95. logger.info("converting keyfile %s" % keyfile)
  96. with open(keyfile, 'r') as f:
  97. data = f.read()
  98. data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1)
  99. keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile))
  100. logger.info("writing borg keyfile to %s" % keyfile)
  101. if not dryrun:
  102. with open(keyfile, 'w') as f:
  103. f.write(data)
  104. def convert_cache(self, dryrun):
  105. """convert caches from attic to borg
  106. those are all hash indexes, so we need to
  107. `s/ATTICIDX/BORG_IDX/` in a few locations:
  108. * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
  109. is the `Repository.get_index_transaction_id()`), which we
  110. should probably update, with a lock, see
  111. `Repository.open()`, which i'm not sure we should use
  112. because it may write data on `Repository.close()`...
  113. * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or
  114. `$HOME/.cache/attic/<repoid>/`), which we could just drop,
  115. but if we'd want to convert, we could open it with the
  116. `Cache.open()`, edit in place and then `Cache.close()` to
  117. make sure we have locking right
  118. """
  119. transaction_id = self.get_index_transaction_id()
  120. if transaction_id is None:
  121. logger.warning('no index file found for repository %s' % self.path)
  122. else:
  123. cache = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8')
  124. logger.info("converting index cache %s" % cache)
  125. if not dryrun:
  126. AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX')
  127. # copy of attic's get_cache_dir()
  128. attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
  129. os.path.join(os.path.expanduser('~'),
  130. '.cache', 'attic'))
  131. attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii'))
  132. borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii'))
  133. def copy_cache_file(path):
  134. """copy the given attic cache path into the borg directory
  135. does nothing if dryrun is True. also expects
  136. attic_cache_dir and borg_cache_dir to be set in the parent
  137. scope, to the directories path including the repository
  138. identifier.
  139. :params path: the basename of the cache file to copy
  140. (example: "files" or "chunks") as a string
  141. :returns: the borg file that was created or None if non
  142. was created.
  143. """
  144. attic_file = os.path.join(attic_cache_dir, path)
  145. if os.path.exists(attic_file):
  146. borg_file = os.path.join(borg_cache_dir, path)
  147. if os.path.exists(borg_file):
  148. logger.warning("borg cache file already exists in %s, skipping conversion of %s" % (borg_file, attic_file))
  149. else:
  150. logger.info("copying attic cache file from %s to %s" % (attic_file, borg_file))
  151. if not dryrun:
  152. shutil.copyfile(attic_file, borg_file)
  153. return borg_file
  154. else:
  155. logger.warning("no %s cache file found in %s" % (path, attic_file))
  156. return None
  157. # XXX: untested, because generating cache files is a PITA, see
  158. # Archiver.do_create() for proof
  159. if os.path.exists(attic_cache_dir):
  160. if not os.path.exists(borg_cache_dir):
  161. os.makedirs(borg_cache_dir)
  162. # file that we don't have a header to convert, just copy
  163. for cache in ['config', 'files']:
  164. copy_cache_file(cache)
  165. # we need to convert the headers of those files, copy first
  166. for cache in ['chunks']:
  167. copied = copy_cache_file(cache)
  168. if copied:
  169. logger.info("converting cache %s" % cache)
  170. if not dryrun:
  171. AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX')
  172. class AtticKeyfileKey(KeyfileKey):
  173. """backwards compatible Attic key file parser"""
  174. FILE_ID = 'ATTIC KEY'
  175. # verbatim copy from attic
  176. @staticmethod
  177. def get_keys_dir():
  178. """Determine where to repository keys and cache"""
  179. return os.environ.get('ATTIC_KEYS_DIR',
  180. os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
  181. @classmethod
  182. def find_key_file(cls, repository):
  183. """copy of attic's `find_key_file`_
  184. this has two small modifications:
  185. 1. it uses the above `get_keys_dir`_ instead of the global one,
  186. assumed to be borg's
  187. 2. it uses `repository.path`_ instead of
  188. `repository._location.canonical_path`_ because we can't
  189. assume the repository has been opened by the archiver yet
  190. """
  191. get_keys_dir = cls.get_keys_dir
  192. id = hexlify(repository.id).decode('ascii')
  193. keys_dir = get_keys_dir()
  194. for name in os.listdir(keys_dir):
  195. filename = os.path.join(keys_dir, name)
  196. with open(filename, 'r') as fd:
  197. line = fd.readline().strip()
  198. if line and line.startswith(cls.FILE_ID) and line[10:] == id:
  199. return filename
  200. raise KeyfileNotFoundError(repository.path, get_keys_dir())