crypto: layout updates, low-level does not deal with IV

encrypt used to "patch" the IV into the header,
decrypt used to fetch it from there.

encrypt now takes the header just "as is" and
also decrypt expects that the IV is already set.

src/borg/crypto/key.py

@@ -713,28 +713,43 @@ class AEADKeyBase(KeyBase):
     """
     Chunks are encrypted and authenticated using some AEAD ciphersuite
 
-    Payload layout: TYPE(1) + SESSIONID(24) + NONCE(12) + MAC(16) + CIPHERTEXT
-                    ^------------- AAD ---------------^
+    Layout: suite:4 keytype:4 reserved:8 messageIV:48 sessionID:192 auth_tag:128 payload:... [bits]
+            ^-------------------- AAD ----------------------------^
+    Offsets:0                 1          2            8             32           48 [bytes]
+
+    suite: 1010b for new AEAD crypto, 0000b is old crypto
+    keytype: see constants.KeyType (suite+keytype)
+    reserved: all-zero, for future use
+    messageIV: a counter starting from 0 for all new encrypted messages of one session
+    sessionID: 192bit random, computed once per session (the session key is derived from this)
+    auth_tag: authentication tag output of the AEAD cipher (computed over payload and AAD)
+    payload: encrypted chunk data
     """
 
-    PAYLOAD_OVERHEAD = 1 + 24 + 12 + 16  # TYPE + SESSIONID + NONCE + MAC
+    PAYLOAD_OVERHEAD = 1 + 1 + 6 + 24 + 16  # [bytes], see Layout
 
     CIPHERSUITE = None  # override in subclass
 
     logically_encrypted = True
 
     def encrypt(self, chunk):
+        # to encrypt new data in this session we use always self.cipher and self.sessionid
         data = self.compressor.compress(chunk)
-        header = self.TYPE_STR + self.sessionid
+        reserved = b'\0'
         iv = self.cipher.next_iv()
+        iv_48bit = iv.to_bytes(6, 'big')
+        header = self.TYPE_STR + reserved + iv_48bit + self.sessionid
         return self.cipher.encrypt(data, header=header, iv=iv)
 
     def decrypt(self, id, data, decompress=True):
+        # to decrypt existing data, we need to get a cipher configured for the sessionid and iv from header
         self.assert_type(data[0], id)
-        sessionid = data[1:13]  # XXX
-        self.init_ciphers(salt=salt, context=context, iv=iv)  # XXX
+        iv_48bit = data[2:8]
+        sessionid = data[8:32]
+        iv = int.from_bytes(iv_48bit, 'big')
+        cipher = self._get_cipher(sessionid, iv)
         try:
-            payload = self.cipher.decrypt(data)
+            payload = cipher.decrypt(data)
         except IntegrityError as e:
             raise IntegrityError(f"Chunk {bin_to_hex(id)}: Could not decrypt [{str(e)}]")
         if not decompress:
@@ -753,15 +768,26 @@ class AEADKeyBase(KeyBase):
         if self.chunk_seed & 0x80000000:
             self.chunk_seed = self.chunk_seed - 0xffffffff - 1
 
-    def init_ciphers(self, salt=b'', context=b'', iv=0):
+    def _get_session_key(self, sessionid):
+        assert len(sessionid) == 24  # 192bit
         key = hkdf_hmac_sha512(
             ikm=self.enc_key + self.enc_hmac_key,
-            salt=salt,
-            info=b'borg-crypto-' + context,  # XXX
+            salt=sessionid,
+            info=b'borg-session-key-' + self.CIPHERSUITE.__name__.encode(),
             output_length=32
         )
-        self.cipher = self.CIPHERSUITE(key=key, header_len=1+24, aad_offset=0)  # XXX
-        self.cipher.set_iv(iv)
+        return key
+
+    def _get_cipher(self, sessionid, iv):
+        assert isinstance(iv, int)
+        key = self._get_session_key(sessionid)
+        cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1+1+6+24, aad_offset=0)
+        return cipher
+
+    def init_ciphers(self, manifest_data=None, iv=0):
+        # in every new session we start with a fresh sessionid and at iv == 0, manifest_data and iv params are ignored
+        self.sessionid = os.urandom(24)
+        self.cipher = self._get_cipher(self.sessionid, iv=0)
 
 
 class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):

src/borg/crypto/low_level.pyx

@@ -424,7 +424,7 @@ ctypedef const EVP_CIPHER * (* CIPHER)()
 
 cdef class _AEAD_BASE:
     # new crypto used in borg >= 1.3
-    # Layout: HEADER + MAC 16 + CT (IV will be put into the header, at the end)
+    # Layout: HEADER + MAC 16 + CT
 
     cdef CIPHER cipher
     cdef EVP_CIPHER_CTX *ctx
@@ -432,7 +432,7 @@ cdef class _AEAD_BASE:
     cdef int cipher_blk_len
     cdef int iv_len
     cdef int aad_offset
-    cdef int header_len_expected  # includes the IV at the end
+    cdef int header_len_expected
     cdef int mac_len
     cdef unsigned char iv[12]
     cdef long long blocks
@@ -448,12 +448,12 @@ cdef class _AEAD_BASE:
 
         :param key: 256bit encrypt-then-mac key
         :param iv: 96bit initialisation vector / nonce
-        :param header_len: expected length of header *without* IV
+        :param header_len: expected length of header
         :param aad_offset: where in the header the authenticated data starts
         """
         assert isinstance(key, bytes) and len(key) == 32
         self.iv_len = sizeof(self.iv)
-        self.header_len_expected = header_len + self.iv_len
+        self.header_len_expected = header_len
         assert aad_offset <= header_len
         self.aad_offset = aad_offset
         self.mac_len = 16
@@ -483,8 +483,7 @@ cdef class _AEAD_BASE:
         if block_count > 2**32:
             raise ValueError('too much data, would overflow internal 32bit counter')
         cdef int ilen = len(data)
-        cdef int hl = len(header)
-        cdef int hlen = hl + self.iv_len
+        cdef int hlen = len(header)
         assert hlen == self.header_len_expected
         cdef int aoffset = self.aad_offset
         cdef int alen = hlen - aoffset
@@ -498,11 +497,9 @@ cdef class _AEAD_BASE:
         cdef Py_buffer hdata = ro_buffer(header)
         try:
             offset = 0
-            for i in range(hl):
+            for i in range(hlen):
                 odata[offset+i] = header[i]
-            offset = hl
-            self.store_iv(odata+offset, self.iv)
-            offset = hlen
+            offset += hlen
             offset += self.mac_len
             rc = EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, NULL, NULL)
             if not rc:
@@ -543,7 +540,6 @@ cdef class _AEAD_BASE:
             raise ValueError('too much data, would overflow internal 32bit counter')
         cdef int ilen = len(envelope)
         cdef int hlen = self.header_len_expected
-        cdef int hl = hlen - self.iv_len
         cdef int aoffset = self.aad_offset
         cdef int alen = hlen - aoffset
         cdef unsigned char *odata = <unsigned char *>PyMem_Malloc(ilen + self.cipher_blk_len)
@@ -555,11 +551,9 @@ cdef class _AEAD_BASE:
         try:
             if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, NULL, NULL):
                 raise CryptoError('EVP_DecryptInit_ex failed')
-            iv = self.fetch_iv(<unsigned char *> idata.buf+hl)
-            self.set_iv(iv)
             if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_SET_IVLEN, self.iv_len, NULL):
                 raise CryptoError('EVP_CIPHER_CTX_ctrl SET IVLEN failed')
-            if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, self.key, iv):
+            if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, self.key, self.iv):
                 raise CryptoError('EVP_DecryptInit_ex failed')
             if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_AEAD_SET_TAG, self.mac_len, <unsigned char *> idata.buf + hlen):
                 raise CryptoError('EVP_CIPHER_CTX_ctrl SET TAG failed')
@@ -604,18 +598,6 @@ cdef class _AEAD_BASE:
         iv = int.from_bytes(self.iv[:self.iv_len], byteorder='big')
         return iv + 1
 
-    cdef fetch_iv(self, unsigned char * iv_in):
-        return iv_in[0:self.iv_len]
-
-    cdef store_iv(self, unsigned char * iv_out, unsigned char * iv):
-        cdef int i
-        for i in range(self.iv_len):
-            iv_out[i] = iv[i]
-
-    def extract_iv(self, envelope):
-        offset = self.header_len_expected - self.iv_len
-        return bytes_to_long(envelope[offset:offset+self.iv_len])
-
 
 cdef class _AES_BASE(_AEAD_BASE):
     def __init__(self, *args, **kwargs):

src/borg/testsuite/crypto.py

@@ -94,7 +94,7 @@ class CryptoTestCase(BaseTestCase):
         key = b'X' * 32
         iv = 0
         data = b'foo' * 10
-        header = b'\x23'
+        header = b'\x23' + iv.to_bytes(12, 'big')
         tests = [
             # (ciphersuite class, exp_mac, exp_cdata)
         ]
@@ -137,7 +137,7 @@ class CryptoTestCase(BaseTestCase):
         key = b'X' * 32
         iv = 0
         data = b'foo' * 10
-        header = b'\x12\x34\x56'
+        header = b'\x12\x34\x56' + iv.to_bytes(12, 'big')
         tests = [
             # (ciphersuite class, exp_mac, exp_cdata)
         ]