fixup: use thread-local buffer

start with 0 bytes length (saves memory in case lz4 is not used).
always grow when a bigger buffer is needed.
avoid per-call reallocation / freeing / garbage.

borg/compress.pyx

@@ -1,3 +1,4 @@
+import threading
 import zlib
 try:
     import lzma
@@ -10,6 +11,17 @@ cdef extern from "lz4.h":
     int LZ4_compressBound(int inputSize) nogil
 
 
+thread_local = threading.local()
+thread_local.buffer = bytes()
+
+
+cdef char *get_buffer(size):
+    size = int(size)
+    if len(thread_local.buffer) < size:
+        thread_local.buffer = bytes(size)
+    return <char *> thread_local.buffer
+
+
 cdef class CompressorBase:
     """
     base class for all (de)compression classes,
@@ -66,11 +78,7 @@ class LZ4(CompressorBase):
     name = 'lz4'
 
     def __init__(self, **kwargs):
-        self.buffer = None
-
-    def _create_buffer(self, size):
-        # we keep a reference to the buffer until this instance is destroyed
-        self.buffer = bytes(int(size))
+        pass
 
     def compress(self, idata):
         if not isinstance(idata, bytes):
@@ -80,8 +88,7 @@ class LZ4(CompressorBase):
         cdef char *source = idata
         cdef char *dest
         osize = LZ4_compressBound(isize)
-        self._create_buffer(osize)
-        dest = self.buffer
+        dest = get_buffer(osize)
         with nogil:
             osize = LZ4_compress_limitedOutput(source, dest, isize, osize)
         if not osize:
@@ -101,8 +108,7 @@ class LZ4(CompressorBase):
         # allocate more if isize * 3 is already bigger, to avoid having to resize often.
         osize = max(int(1.1 * 2**23), isize * 3)
         while True:
-            self._create_buffer(osize)
-            dest = self.buffer
+            dest = get_buffer(osize)
             with nogil:
                 rsize = LZ4_decompress_safe(source, dest, isize, osize)
             if rsize >= 0: