key.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751
  1. from binascii import hexlify, a2b_base64, b2a_base64
  2. from getpass import getpass
  3. import os
  4. import msgpack
  5. import textwrap
  6. from collections import namedtuple
  7. import hmac
  8. from hashlib import sha256, sha512
  9. import zlib
  10. try:
  11. import lzma # python >= 3.3
  12. except ImportError:
  13. try:
  14. from backports import lzma # backports.lzma from pypi
  15. except ImportError:
  16. lzma = None
  17. from attic.crypto import pbkdf2_sha256, get_random_bytes, AES, AES_CTR_MODE, AES_GCM_MODE, \
  18. bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
  19. from attic.helpers import IntegrityError, get_keys_dir, Error
  20. # we do not store the full IV on disk, as the upper 8 bytes are expected to be
  21. # zero anyway as the full IV is a 128bit counter. PREFIX are the upper 8 bytes,
  22. # stored_iv are the lower 8 Bytes.
  23. PREFIX = b'\0' * 8
  24. Meta = namedtuple('Meta', 'compr_type, key_type, mac_type, cipher_type, hmac, stored_iv')
  25. class UnsupportedPayloadError(Error):
  26. """Unsupported payload type {}. A newer version is required to access this repository.
  27. """
  28. class sha512_256(object): # note: can't subclass sha512
  29. """sha512, but digest truncated to 256bit - faster than sha256 on 64bit platforms"""
  30. digestsize = digest_size = 32
  31. block_size = 64
  32. def __init__(self, data=None):
  33. self.name = 'sha512-256'
  34. self._h = sha512()
  35. if data:
  36. self.update(data)
  37. def update(self, data):
  38. self._h.update(data)
  39. def digest(self):
  40. return self._h.digest()[:self.digest_size]
  41. def hexdigest(self):
  42. return self._h.hexdigest()[:self.digest_size * 2]
  43. def copy(self):
  44. new = sha512_256.__new__(sha512_256)
  45. new._h = self._h.copy()
  46. return new
  47. class HMAC(hmac.HMAC):
  48. """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews
  49. """
  50. def update(self, msg):
  51. self.inner.update(msg)
  52. # HASH / MAC stuff below all has a mac-like interface, so it can be used in the same way.
  53. # special case: hashes do not use keys (and thus, do not sign/authenticate)
  54. class SHA256(object): # note: can't subclass sha256
  55. TYPE = 0
  56. def __init__(self, key, data=b''):
  57. # signature is like for a MAC, we ignore the key as this is a simple hash
  58. if key is not None:
  59. raise Exception("use a HMAC if you have a key")
  60. self.h = sha256(data)
  61. def update(self, data):
  62. self.h.update(data)
  63. def digest(self):
  64. return self.h.digest()
  65. def hexdigest(self):
  66. return self.h.hexdigest()
  67. class SHA512_256(sha512_256):
  68. """sha512, but digest truncated to 256bit - faster than sha256 on 64bit platforms"""
  69. TYPE = 1
  70. def __init__(self, key, data):
  71. # signature is like for a MAC, we ignore the key as this is a simple hash
  72. if key is not None:
  73. raise Exception("use a HMAC if you have a key")
  74. super().__init__(data)
  75. class GHASH:
  76. TYPE = 2
  77. def __init__(self, key, data):
  78. # signature is like for a MAC, we ignore the key as this is a simple hash
  79. if key is not None:
  80. raise Exception("use a MAC if you have a key")
  81. self.key = b'\0' * 32
  82. self.data = data
  83. def digest(self):
  84. mac_cipher = AES(mode=AES_GCM_MODE, is_encrypt=True, key=self.key, iv=b'\0' * 16)
  85. # GMAC = aes-gcm with all data as AAD, no data as to-be-encrypted data
  86. mac_cipher.add(bytes(self.data))
  87. tag, _ = mac_cipher.compute_tag_and_encrypt(b'')
  88. return tag
  89. class HMAC_SHA256(HMAC):
  90. TYPE = 10
  91. def __init__(self, key, data):
  92. if key is None:
  93. raise Exception("do not use HMAC if you don't have a key")
  94. super().__init__(key, data, sha256)
  95. class HMAC_SHA512_256(HMAC):
  96. TYPE = 11
  97. def __init__(self, key, data):
  98. if key is None:
  99. raise Exception("do not use HMAC if you don't have a key")
  100. super().__init__(key, data, sha512_256)
  101. class GMAC(GHASH):
  102. TYPE = 20
  103. def __init__(self, key, data):
  104. super().__init__(None, data)
  105. if key is None:
  106. raise Exception("do not use GMAC if you don't have a key")
  107. self.key = key
  108. # defaults are optimized for speed on modern CPUs with AES hw support
  109. HASH_DEFAULT = GHASH.TYPE
  110. MAC_DEFAULT = GMAC.TYPE
  111. # compressor classes, all same interface
  112. # special case: zlib level 0 is "no compression"
  113. class NullCompressor(object): # uses 0 in the mapping
  114. TYPE = 0
  115. def compress(self, data):
  116. return bytes(data)
  117. def decompress(self, data):
  118. return bytes(data)
  119. class ZlibCompressor(object): # uses 1..9 in the mapping
  120. TYPE = 0
  121. LEVELS = range(10)
  122. def compress(self, data):
  123. level = self.TYPE - ZlibCompressor.TYPE
  124. return zlib.compress(data, level)
  125. def decompress(self, data):
  126. return zlib.decompress(data)
  127. class LzmaCompressor(object): # uses 10..19 in the mapping
  128. TYPE = 10
  129. PRESETS = range(10)
  130. def __init__(self):
  131. if lzma is None:
  132. raise NotImplemented("lzma compression needs Python >= 3.3 or backports.lzma from PyPi")
  133. def compress(self, data):
  134. preset = self.TYPE - LzmaCompressor.TYPE
  135. return lzma.compress(data, preset=preset)
  136. def decompress(self, data):
  137. return lzma.decompress(data)
  138. # default is optimized for speed
  139. COMPR_DEFAULT = NullCompressor.TYPE # no compression
  140. # ciphers - AEAD (authenticated encryption with assoc. data) style interface
  141. # special case: PLAIN dummy does not encrypt / authenticate
  142. class PLAIN:
  143. TYPE = 0
  144. def __init__(self, **kw):
  145. pass
  146. def compute_tag_and_encrypt(self, data):
  147. return b'', b'', data
  148. def check_tag_and_decrypt(self, tag, iv_last8, data):
  149. return data
  150. class AES_CTR_HMAC:
  151. TYPE = 1
  152. def __init__(self, enc_key=b'\0' * 32, enc_iv=b'\0' * 16, enc_hmac_key=b'\0' * 32, **kw):
  153. self.hmac_key = enc_hmac_key
  154. self.enc_iv = enc_iv
  155. self.enc_cipher = AES(mode=AES_CTR_MODE, is_encrypt=True, key=enc_key, iv=enc_iv)
  156. self.dec_cipher = AES(mode=AES_CTR_MODE, is_encrypt=False, key=enc_key)
  157. def compute_tag_and_encrypt(self, data):
  158. self.enc_cipher.reset(iv=self.enc_iv)
  159. iv_last8 = self.enc_iv[8:]
  160. _, data = self.enc_cipher.compute_tag_and_encrypt(data)
  161. # increase the IV (counter) value so same value is never used twice
  162. current_iv = bytes_to_long(iv_last8)
  163. self.enc_iv = PREFIX + long_to_bytes(current_iv + num_aes_blocks(len(data)))
  164. tag = HMAC(self.hmac_key, iv_last8 + data, sha256).digest() # XXX mac / hash flexibility
  165. return tag, iv_last8, data
  166. def check_tag_and_decrypt(self, tag, iv_last8, data):
  167. iv = PREFIX + iv_last8
  168. if HMAC(self.hmac_key, iv_last8 + data, sha256).digest() != tag:
  169. raise IntegrityError('Encryption envelope checksum mismatch')
  170. self.dec_cipher.reset(iv=iv)
  171. data = self.dec_cipher.check_tag_and_decrypt(None, data)
  172. return data
  173. class AES_GCM:
  174. TYPE = 2
  175. def __init__(self, enc_key=b'\0' * 32, enc_iv=b'\0' * 16, **kw):
  176. # note: hmac_key is not used for aes-gcm, it does aes+gmac in 1 pass
  177. self.enc_iv = enc_iv
  178. self.enc_cipher = AES(mode=AES_GCM_MODE, is_encrypt=True, key=enc_key, iv=enc_iv)
  179. self.dec_cipher = AES(mode=AES_GCM_MODE, is_encrypt=False, key=enc_key)
  180. def compute_tag_and_encrypt(self, data):
  181. self.enc_cipher.reset(iv=self.enc_iv)
  182. iv_last8 = self.enc_iv[8:]
  183. self.enc_cipher.add(iv_last8)
  184. tag, data = self.enc_cipher.compute_tag_and_encrypt(data)
  185. # increase the IV (counter) value so same value is never used twice
  186. current_iv = bytes_to_long(iv_last8)
  187. self.enc_iv = PREFIX + long_to_bytes(current_iv + num_aes_blocks(len(data)))
  188. return tag, iv_last8, data
  189. def check_tag_and_decrypt(self, tag, iv_last8, data):
  190. iv = PREFIX + iv_last8
  191. self.dec_cipher.reset(iv=iv)
  192. self.dec_cipher.add(iv_last8)
  193. try:
  194. data = self.dec_cipher.check_tag_and_decrypt(tag, data)
  195. except Exception:
  196. raise IntegrityError('Encryption envelope checksum mismatch')
  197. return data
  198. # cipher default is optimized for speed on modern CPUs with AES hw support
  199. PLAIN_DEFAULT = PLAIN.TYPE
  200. CIPHER_DEFAULT = AES_GCM.TYPE
  201. # misc. types of keys
  202. # special case: no keys (thus: no encryption, no signing/authentication)
  203. class KeyBase(object):
  204. TYPE = 0x00 # override in derived classes
  205. def __init__(self, compressor_cls, maccer_cls, cipher_cls):
  206. self.compressor = compressor_cls()
  207. self.maccer_cls = maccer_cls # hasher/maccer used by id_hash
  208. self.cipher_cls = cipher_cls # plaintext dummy or AEAD cipher
  209. self.cipher = cipher_cls()
  210. self.id_key = None
  211. def id_hash(self, data):
  212. """Return a HASH (no id_key) or a MAC (using the "id_key" key)
  213. XXX do we need a cryptographic hash function here or is a keyed hash
  214. function like GMAC / GHASH good enough? See NIST SP 800-38D.
  215. IMPORTANT: in 1 repo, there should be only 1 kind of id_hash, otherwise
  216. data hashed/maced with one id_hash might result in same ID as already
  217. exists in the repo for other data created with another id_hash method.
  218. somehow unlikely considering 128 or 256bits, but still.
  219. """
  220. return self.maccer_cls(self.id_key, data).digest()
  221. def encrypt(self, data):
  222. data = self.compressor.compress(data)
  223. tag, iv_last8, data = self.cipher.compute_tag_and_encrypt(data)
  224. meta = Meta(compr_type=self.compressor.TYPE, key_type=self.TYPE,
  225. mac_type=self.maccer_cls.TYPE, cipher_type=self.cipher.TYPE,
  226. hmac=tag, stored_iv=iv_last8)
  227. return generate(meta, data)
  228. def decrypt(self, id, data):
  229. meta, data = parser(data)
  230. compressor, keyer, maccer, cipher = get_implementations(meta)
  231. assert isinstance(self, keyer)
  232. assert self.maccer_cls is maccer
  233. assert self.cipher_cls is cipher
  234. data = self.cipher.check_tag_and_decrypt(meta.hmac, meta.stored_iv, data)
  235. data = self.compressor.decompress(data)
  236. if id and self.id_hash(data) != id:
  237. raise IntegrityError('Chunk id verification failed')
  238. return data
  239. class PlaintextKey(KeyBase):
  240. TYPE = 0x02
  241. chunk_seed = 0
  242. @classmethod
  243. def create(cls, repository, args):
  244. print('Encryption NOT enabled.\nUse the "--encryption=passphrase|keyfile" to enable encryption.')
  245. compressor = compressor_creator(args)
  246. maccer = maccer_creator(args, cls)
  247. cipher = cipher_creator(args, cls)
  248. return cls(compressor, maccer, cipher)
  249. @classmethod
  250. def detect(cls, repository, manifest_data):
  251. meta, data = parser(manifest_data)
  252. compressor, keyer, maccer, cipher = get_implementations(meta)
  253. return cls(compressor, maccer, cipher)
  254. class AESKeyBase(KeyBase):
  255. """Common base class shared by KeyfileKey and PassphraseKey
  256. Chunks are encrypted using 256bit AES in CTR or GCM mode.
  257. Chunks are authenticated by a GCM GMAC or a HMAC.
  258. Payload layout: TYPE(1) + MAC(32) + NONCE(8) + CIPHERTEXT
  259. To reduce payload size only 8 bytes of the 16 bytes nonce is saved
  260. in the payload, the first 8 bytes are always zeros. This does not
  261. affect security but limits the maximum repository capacity to
  262. only 295 exabytes!
  263. """
  264. def extract_nonce(self, payload):
  265. meta, data = parser(payload)
  266. nonce = bytes_to_long(meta.stored_iv)
  267. return nonce
  268. def init_from_random_data(self, data):
  269. self.enc_key = data[0:32]
  270. self.enc_hmac_key = data[32:64]
  271. self.id_key = data[64:96]
  272. self.chunk_seed = bytes_to_int(data[96:100])
  273. # Convert to signed int32
  274. if self.chunk_seed & 0x80000000:
  275. self.chunk_seed = self.chunk_seed - 0xffffffff - 1
  276. def init_ciphers(self, enc_iv=b'\0' * 16):
  277. self.cipher = self.cipher_cls(enc_key=self.enc_key, enc_iv=enc_iv,
  278. enc_hmac_key=self.enc_hmac_key)
  279. @property
  280. def enc_iv(self):
  281. return self.cipher.enc_iv
  282. class PassphraseKey(AESKeyBase):
  283. TYPE = 0x01
  284. iterations = 100000
  285. @classmethod
  286. def create(cls, repository, args):
  287. compressor = compressor_creator(args)
  288. maccer = maccer_creator(args, cls)
  289. cipher = cipher_creator(args, cls)
  290. key = cls(compressor, maccer, cipher)
  291. passphrase = os.environ.get('ATTIC_PASSPHRASE')
  292. if passphrase is not None:
  293. passphrase2 = passphrase
  294. else:
  295. passphrase, passphrase2 = 1, 2
  296. while passphrase != passphrase2:
  297. passphrase = getpass('Enter passphrase: ')
  298. if not passphrase:
  299. print('Passphrase must not be blank')
  300. continue
  301. passphrase2 = getpass('Enter same passphrase again: ')
  302. if passphrase != passphrase2:
  303. print('Passphrases do not match')
  304. key.init(repository, passphrase)
  305. if passphrase:
  306. print('Remember your passphrase. Your data will be inaccessible without it.')
  307. return key
  308. @classmethod
  309. def detect(cls, repository, manifest_data):
  310. prompt = 'Enter passphrase for %s: ' % repository._location.orig
  311. meta, data = parser(manifest_data)
  312. compressor, keyer, maccer, cipher = get_implementations(meta)
  313. key = cls(compressor, maccer, cipher)
  314. passphrase = os.environ.get('ATTIC_PASSPHRASE')
  315. if passphrase is None:
  316. passphrase = getpass(prompt)
  317. while True:
  318. key.init(repository, passphrase)
  319. try:
  320. key.decrypt(None, manifest_data)
  321. num_blocks = num_aes_blocks(len(data))
  322. key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
  323. return key
  324. except IntegrityError:
  325. passphrase = getpass(prompt)
  326. def change_passphrase(self):
  327. class ImmutablePassphraseError(Error):
  328. """The passphrase for this encryption key type can't be changed."""
  329. raise ImmutablePassphraseError
  330. def init(self, repository, passphrase):
  331. self.init_from_random_data(pbkdf2_sha256(passphrase.encode('utf-8'), repository.id, self.iterations, 100))
  332. self.init_ciphers()
  333. class KeyfileKey(AESKeyBase):
  334. FILE_ID = 'ATTIC KEY'
  335. TYPE = 0x00
  336. @classmethod
  337. def detect(cls, repository, manifest_data):
  338. meta, data = parser(manifest_data)
  339. compressor, keyer, maccer, cipher = get_implementations(meta)
  340. key = cls(compressor, maccer, cipher)
  341. path = cls.find_key_file(repository)
  342. prompt = 'Enter passphrase for key file %s: ' % path
  343. passphrase = os.environ.get('ATTIC_PASSPHRASE', '')
  344. while not key.load(path, passphrase):
  345. passphrase = getpass(prompt)
  346. num_blocks = num_aes_blocks(len(data))
  347. key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
  348. return key
  349. @classmethod
  350. def find_key_file(cls, repository):
  351. id = hexlify(repository.id).decode('ascii')
  352. keys_dir = get_keys_dir()
  353. for name in os.listdir(keys_dir):
  354. filename = os.path.join(keys_dir, name)
  355. with open(filename, 'r') as fd:
  356. line = fd.readline().strip()
  357. if line and line.startswith(cls.FILE_ID) and line[10:] == id:
  358. return filename
  359. raise Exception('Key file for repository with ID %s not found' % id)
  360. def load(self, filename, passphrase):
  361. with open(filename, 'r') as fd:
  362. cdata = a2b_base64(''.join(fd.readlines()[1:]).encode('ascii')) # .encode needed for Python 3.[0-2]
  363. data = self.decrypt_key_file(cdata, passphrase)
  364. if data:
  365. key = msgpack.unpackb(data)
  366. if key[b'version'] != 1:
  367. raise IntegrityError('Invalid key file header')
  368. self.repository_id = key[b'repository_id']
  369. self.enc_key = key[b'enc_key']
  370. self.enc_hmac_key = key[b'enc_hmac_key']
  371. self.id_key = key[b'id_key']
  372. self.chunk_seed = key[b'chunk_seed']
  373. self.path = filename
  374. return True
  375. def decrypt_key_file(self, data, passphrase):
  376. d = msgpack.unpackb(data)
  377. assert d[b'version'] == 1
  378. assert d[b'algorithm'] == b'gmac'
  379. key = pbkdf2_sha256(passphrase.encode('utf-8'), d[b'salt'], d[b'iterations'], 32)
  380. try:
  381. cipher = AES(mode=AES_GCM_MODE, is_encrypt=False, key=key, iv=b'\0'*16)
  382. data = cipher.check_tag_and_decrypt(d[b'hash'], d[b'data'])
  383. return data
  384. except Exception:
  385. return None
  386. def encrypt_key_file(self, data, passphrase):
  387. salt = get_random_bytes(32)
  388. iterations = 100000
  389. key = pbkdf2_sha256(passphrase.encode('utf-8'), salt, iterations, 32)
  390. cipher = AES(mode=AES_GCM_MODE, is_encrypt=True, key=key, iv=b'\0'*16)
  391. tag, cdata = cipher.compute_tag_and_encrypt(data)
  392. d = {
  393. 'version': 1,
  394. 'salt': salt,
  395. 'iterations': iterations,
  396. 'algorithm': 'gmac',
  397. 'hash': tag,
  398. 'data': cdata,
  399. }
  400. return msgpack.packb(d)
  401. def save(self, path, passphrase):
  402. key = {
  403. 'version': 1,
  404. 'repository_id': self.repository_id,
  405. 'enc_key': self.enc_key,
  406. 'enc_hmac_key': self.enc_hmac_key,
  407. 'id_key': self.id_key,
  408. 'chunk_seed': self.chunk_seed,
  409. }
  410. data = self.encrypt_key_file(msgpack.packb(key), passphrase)
  411. with open(path, 'w') as fd:
  412. fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
  413. fd.write('\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii'))))
  414. fd.write('\n')
  415. self.path = path
  416. def change_passphrase(self):
  417. passphrase, passphrase2 = 1, 2
  418. while passphrase != passphrase2:
  419. passphrase = getpass('New passphrase: ')
  420. passphrase2 = getpass('Enter same passphrase again: ')
  421. if passphrase != passphrase2:
  422. print('Passphrases do not match')
  423. self.save(self.path, passphrase)
  424. print('Key file "%s" updated' % self.path)
  425. @classmethod
  426. def create(cls, repository, args):
  427. filename = args.repository.to_key_filename()
  428. path = filename
  429. i = 1
  430. while os.path.exists(path):
  431. i += 1
  432. path = filename + '.%d' % i
  433. passphrase = os.environ.get('ATTIC_PASSPHRASE')
  434. if passphrase is not None:
  435. passphrase2 = passphrase
  436. else:
  437. passphrase, passphrase2 = 1, 2
  438. while passphrase != passphrase2:
  439. passphrase = getpass('Enter passphrase (empty for no passphrase):')
  440. passphrase2 = getpass('Enter same passphrase again: ')
  441. if passphrase != passphrase2:
  442. print('Passphrases do not match')
  443. compressor = compressor_creator(args)
  444. maccer = maccer_creator(args, cls)
  445. cipher = cipher_creator(args, cls)
  446. key = cls(compressor, maccer, cipher)
  447. key.repository_id = repository.id
  448. key.init_from_random_data(get_random_bytes(100))
  449. key.init_ciphers()
  450. key.save(path, passphrase)
  451. print('Key file "%s" created.' % key.path)
  452. print('Keep this file safe. Your data will be inaccessible without it.')
  453. return key
  454. # note: key 0 nicely maps to a zlib compressor with level 0 which means "no compression"
  455. compressor_mapping = {}
  456. for level in ZlibCompressor.LEVELS:
  457. compressor_mapping[ZlibCompressor.TYPE + level] = \
  458. type('ZlibCompressorLevel%d' % level, (ZlibCompressor, ), dict(TYPE=ZlibCompressor.TYPE + level))
  459. for preset in LzmaCompressor.PRESETS:
  460. compressor_mapping[LzmaCompressor.TYPE + preset] = \
  461. type('LzmaCompressorPreset%d' % preset, (LzmaCompressor, ), dict(TYPE=LzmaCompressor.TYPE + preset))
  462. # overwrite 0 with NullCompressor
  463. compressor_mapping[NullCompressor.TYPE] = NullCompressor
  464. keyer_mapping = {
  465. KeyfileKey.TYPE: KeyfileKey,
  466. PassphraseKey.TYPE: PassphraseKey,
  467. PlaintextKey.TYPE: PlaintextKey,
  468. }
  469. maccer_mapping = {
  470. # simple hashes, not MACs (but MAC-like class __init__ method signature):
  471. SHA256.TYPE: SHA256,
  472. SHA512_256.TYPE: SHA512_256,
  473. GHASH.TYPE: GHASH,
  474. # MACs:
  475. HMAC_SHA256.TYPE: HMAC_SHA256,
  476. HMAC_SHA512_256.TYPE: HMAC_SHA512_256,
  477. GMAC.TYPE: GMAC,
  478. }
  479. cipher_mapping = {
  480. # no cipher (but cipher-like class __init__ method signature):
  481. PLAIN.TYPE: PLAIN,
  482. # AEAD cipher implementations
  483. AES_CTR_HMAC.TYPE: AES_CTR_HMAC,
  484. AES_GCM.TYPE: AES_GCM,
  485. }
  486. def get_implementations(meta):
  487. try:
  488. compressor = compressor_mapping[meta.compr_type]
  489. keyer = keyer_mapping[meta.key_type]
  490. maccer = maccer_mapping[meta.mac_type]
  491. cipher = cipher_mapping[meta.cipher_type]
  492. except KeyError:
  493. raise UnsupportedPayloadError("compr_type %x key_type %x mac_type %x cipher_type %x" % (
  494. meta.compr_type, meta.key_type, meta.mac_type, meta.cipher_type))
  495. return compressor, keyer, maccer, cipher
  496. def legacy_parser(all_data, key_type): # all rather hardcoded
  497. """
  498. Payload layout:
  499. no encryption: TYPE(1) + data
  500. with encryption: TYPE(1) + HMAC(32) + NONCE(8) + data
  501. data is compressed with zlib level 6 and (in the 2nd case) encrypted.
  502. To reduce payload size only 8 bytes of the 16 bytes nonce is saved
  503. in the payload, the first 8 bytes are always zeros. This does not
  504. affect security but limits the maximum repository capacity to
  505. only 295 exabytes!
  506. """
  507. offset = 1
  508. if key_type == PlaintextKey.TYPE:
  509. hmac = None
  510. iv = stored_iv = None
  511. data = all_data[offset:]
  512. else:
  513. hmac = all_data[offset:offset+32]
  514. stored_iv = all_data[offset+32:offset+40]
  515. data = all_data[offset+40:]
  516. meta = Meta(compr_type=6, key_type=key_type,
  517. mac_type=HMAC_SHA256.TYPE, cipher_type=AES_CTR_HMAC.TYPE,
  518. hmac=hmac, stored_iv=stored_iv)
  519. return meta, data
  520. def parser00(all_data):
  521. return legacy_parser(all_data, KeyfileKey.TYPE)
  522. def parser01(all_data):
  523. return legacy_parser(all_data, PassphraseKey.TYPE)
  524. def parser02(all_data):
  525. return legacy_parser(all_data, PlaintextKey.TYPE)
  526. def parser03(all_data): # new & flexible
  527. """
  528. Payload layout:
  529. always: TYPE(1) + MSGPACK((meta, data))
  530. meta is a Meta namedtuple and contains all required information about data.
  531. data is maybe compressed (see meta) and maybe encrypted (see meta).
  532. """
  533. max_len = 10000000 # XXX formula?
  534. unpacker = msgpack.Unpacker(
  535. use_list=False,
  536. # avoid memory allocation issues causes by tampered input data.
  537. max_buffer_size=max_len, # does not work in 0.4.6 unpackb C implementation
  538. max_array_len=10, # meta_tuple
  539. max_bin_len=max_len, # data
  540. max_str_len=0, # not used yet
  541. max_map_len=0, # not used yet
  542. max_ext_len=0, # not used yet
  543. )
  544. unpacker.feed(all_data[1:])
  545. meta_tuple, data = unpacker.unpack()
  546. meta = Meta(*meta_tuple)
  547. return meta, data
  548. def parser(data):
  549. parser_mapping = {
  550. 0x00: parser00,
  551. 0x01: parser01,
  552. 0x02: parser02,
  553. 0x03: parser03,
  554. }
  555. header_type = data[0]
  556. parser_func = parser_mapping[header_type]
  557. return parser_func(data)
  558. def key_factory(repository, manifest_data):
  559. meta, data = parser(manifest_data)
  560. compressor, keyer, maccer, cipher = get_implementations(meta)
  561. return keyer.detect(repository, manifest_data)
  562. def generate(meta, data):
  563. # always create new-style 0x03 format
  564. return b'\x03' + msgpack.packb((meta, data), use_bin_type=True)
  565. def compressor_creator(args):
  566. # args == None is used by unit tests
  567. compression = COMPR_DEFAULT if args is None else args.compression
  568. compressor = compressor_mapping.get(compression)
  569. if compressor is None:
  570. raise NotImplementedError("no compression %d" % args.compression)
  571. return compressor
  572. def key_creator(repository, args):
  573. if args.encryption == 'keyfile':
  574. return KeyfileKey.create(repository, args)
  575. if args.encryption == 'passphrase':
  576. return PassphraseKey.create(repository, args)
  577. if args.encryption == 'none':
  578. return PlaintextKey.create(repository, args)
  579. raise NotImplemented("no encryption %s" % args.encryption)
  580. def maccer_creator(args, key_cls):
  581. # args == None is used by unit tests
  582. mac = None if args is None else args.mac
  583. if mac is None:
  584. if key_cls is PlaintextKey:
  585. mac = HASH_DEFAULT
  586. elif key_cls in (KeyfileKey, PassphraseKey):
  587. mac = MAC_DEFAULT
  588. else:
  589. raise NotImplementedError("unknown key class")
  590. maccer = maccer_mapping.get(mac)
  591. if maccer is None:
  592. raise NotImplementedError("no mac %d" % args.mac)
  593. return maccer
  594. def cipher_creator(args, key_cls):
  595. # args == None is used by unit tests
  596. cipher = None if args is None else args.cipher
  597. if cipher is None:
  598. if key_cls is PlaintextKey:
  599. cipher = PLAIN_DEFAULT
  600. elif key_cls in (KeyfileKey, PassphraseKey):
  601. cipher = CIPHER_DEFAULT
  602. else:
  603. raise NotImplementedError("unknown key class")
  604. cipher = cipher_mapping.get(cipher)
  605. if cipher is None:
  606. raise NotImplementedError("no cipher %d" % args.cipher)
  607. return cipher