Merge pull request #1193 from ThomasWaldmann/openssl-1.0-1.1-compat

OpenSSL 1.0 and 1.1 compatibility

borg/crypto.pyx

@@ -16,13 +16,12 @@ cdef extern from "openssl/evp.h":
     ctypedef struct EVP_CIPHER:
         pass
     ctypedef struct EVP_CIPHER_CTX:
-        unsigned char *iv
         pass
     ctypedef struct ENGINE:
         pass
     const EVP_CIPHER *EVP_aes_256_ctr()
-    void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a)
-    void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a)
+    EVP_CIPHER_CTX *EVP_CIPHER_CTX_new()
+    void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a)
 
     int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
                            const unsigned char *key, const unsigned char *iv)
@@ -40,12 +39,40 @@ import struct
 
 _int = struct.Struct('>I')
 _long = struct.Struct('>Q')
+_2long = struct.Struct('>QQ')
 
 bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0]
 bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0]
 long_to_bytes = lambda x: _long.pack(x)
 
 
+def bytes16_to_int(b, offset=0):
+    h, l = _2long.unpack_from(b, offset)
+    return (h << 64) + l
+
+
+def int_to_bytes16(i):
+    max_uint64 = 0xffffffffffffffff
+    l = i & max_uint64
+    h = (i >> 64) & max_uint64
+    return _2long.pack(h, l)
+
+
+def increment_iv(iv, amount=1):
+    """
+    Increment the IV by the given amount (default 1).
+
+    :param iv: input IV, 16 bytes (128 bit)
+    :param amount: increment value
+    :return: input_IV + amount, 16 bytes (128 bit)
+    """
+    assert len(iv) == 16
+    iv = bytes16_to_int(iv)
+    iv += amount
+    iv = int_to_bytes16(iv)
+    return iv
+
+
 def num_aes_blocks(int length):
     """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data.
        Note: this is only correct for modes without padding, like AES-CTR.
@@ -56,24 +83,26 @@ def num_aes_blocks(int length):
 cdef class AES:
     """A thin wrapper around the OpenSSL EVP cipher API
     """
-    cdef EVP_CIPHER_CTX ctx
+    cdef EVP_CIPHER_CTX *ctx
     cdef int is_encrypt
+    cdef unsigned char iv_orig[16]
+    cdef int blocks
 
     def __cinit__(self, is_encrypt, key, iv=None):
-        EVP_CIPHER_CTX_init(&self.ctx)
+        self.ctx = EVP_CIPHER_CTX_new()
         self.is_encrypt = is_encrypt
         # Set cipher type and mode
         cipher_mode = EVP_aes_256_ctr()
         if self.is_encrypt:
-            if not EVP_EncryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL):
+            if not EVP_EncryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL):
                 raise Exception('EVP_EncryptInit_ex failed')
         else:  # decrypt
-            if not EVP_DecryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL):
+            if not EVP_DecryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL):
                 raise Exception('EVP_DecryptInit_ex failed')
         self.reset(key, iv)
 
     def __dealloc__(self):
-        EVP_CIPHER_CTX_cleanup(&self.ctx)
+        EVP_CIPHER_CTX_free(self.ctx)
 
     def reset(self, key=None, iv=None):
         cdef const unsigned char *key2 = NULL
@@ -82,17 +111,21 @@ cdef class AES:
             key2 = key
         if iv:
             iv2 = iv
+            assert isinstance(iv, bytes) and len(iv) == 16
+            for i in range(16):
+                self.iv_orig[i] = iv[i]
+            self.blocks = 0  # number of AES blocks encrypted starting with iv_orig
         # Initialise key and IV
         if self.is_encrypt:
-            if not EVP_EncryptInit_ex(&self.ctx, NULL, NULL, key2, iv2):
+            if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2):
                 raise Exception('EVP_EncryptInit_ex failed')
         else:  # decrypt
-            if not EVP_DecryptInit_ex(&self.ctx, NULL, NULL, key2, iv2):
+            if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, key2, iv2):
                 raise Exception('EVP_DecryptInit_ex failed')
 
     @property
     def iv(self):
-        return self.ctx.iv[:16]
+        return increment_iv(self.iv_orig[:16], self.blocks)
 
     def encrypt(self, data):
         cdef int inl = len(data)
@@ -103,12 +136,13 @@ cdef class AES:
         if not out:
             raise MemoryError
         try:
-            if not EVP_EncryptUpdate(&self.ctx, out, &outl, data, inl):
+            if not EVP_EncryptUpdate(self.ctx, out, &outl, data, inl):
                 raise Exception('EVP_EncryptUpdate failed')
             ctl = outl
-            if not EVP_EncryptFinal_ex(&self.ctx, out+ctl, &outl):
+            if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl):
                 raise Exception('EVP_EncryptFinal failed')
             ctl += outl
+            self.blocks += num_aes_blocks(ctl)
             return out[:ctl]
         finally:
             free(out)
@@ -124,15 +158,16 @@ cdef class AES:
         if not out:
             raise MemoryError
         try:
-            if not EVP_DecryptUpdate(&self.ctx, out, &outl, data, inl):
+            if not EVP_DecryptUpdate(self.ctx, out, &outl, data, inl):
                 raise Exception('EVP_DecryptUpdate failed')
             ptl = outl
-            if EVP_DecryptFinal_ex(&self.ctx, out+ptl, &outl) <= 0:
+            if EVP_DecryptFinal_ex(self.ctx, out+ptl, &outl) <= 0:
                 # this error check is very important for modes with padding or
                 # authentication. for them, a failure here means corrupted data.
                 # CTR mode does not use padding nor authentication.
                 raise Exception('EVP_DecryptFinal failed')
             ptl += outl
+            self.blocks += num_aes_blocks(inl)
             return out[:ptl]
         finally:
             free(out)

borg/testsuite/crypto.py

@@ -1,6 +1,7 @@
 from binascii import hexlify
 
 from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes
+from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16
 from . import BaseTestCase
 
 
@@ -13,6 +14,27 @@ class CryptoTestCase(BaseTestCase):
         self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1)
         self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1')
 
+    def test_bytes16_to_int(self):
+        self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1'), 1)
+        self.assert_equal(int_to_bytes16(1), b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1')
+        self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0'), 2 ** 64)
+        self.assert_equal(int_to_bytes16(2 ** 64), b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0')
+
+    def test_increment_iv(self):
+        iv0 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'
+        iv1 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1'
+        iv2 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\2'
+        self.assert_equal(increment_iv(iv0, 0), iv0)
+        self.assert_equal(increment_iv(iv0, 1), iv1)
+        self.assert_equal(increment_iv(iv0, 2), iv2)
+        iva = b'\0\0\0\0\0\0\0\0\xff\xff\xff\xff\xff\xff\xff\xff'
+        ivb = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x00'
+        ivc = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x01'
+        self.assert_equal(increment_iv(iva, 0), iva)
+        self.assert_equal(increment_iv(iva, 1), ivb)
+        self.assert_equal(increment_iv(iva, 2), ivc)
+        self.assert_equal(increment_iv(iv0, 2**64), ivb)
+
     def test_aes(self):
         key = b'X' * 32
         data = b'foo' * 10

requirements.d/attic.txt

@@ -0,0 +1,5 @@
+# Please note:
+# attic only builds using OpenSSL 1.0.x, it can not be installed using OpenSSL >= 1.1.0.
+# If attic is not installed, our unit tests will just skip the tests that require attic.
+attic
+

tox.ini

@@ -10,7 +10,7 @@ envlist = py{34,35},flake8
 changedir = {toxworkdir}
 deps =
      -rrequirements.d/development.txt
-     attic
+     -rrequirements.d/attic.txt
 commands = py.test --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *