get rid of the CacheSynchronizer

Lots of low-level code written back then to optimize runtime of some
functions.

We'll solve this differently by doing less stats, esp. if it is expensive to compute.

scripts/fuzz-cache-sync/HOWTO

@@ -1,10 +0,0 @@
-- Install AFL and the requirements for LLVM mode (see docs)
-- Compile the fuzzing target, e.g.
-
-  AFL_HARDEN=1 afl-clang-fast main.c -o fuzz-target -O3
-
-  (other options, like using ASan or MSan are possible as well)
-- Add additional test cases to testcase_dir
-- Run afl, easiest (but inefficient) way;
-
-  afl-fuzz -i testcase_dir -o findings_dir ./fuzz-target

scripts/fuzz-cache-sync/main.c

@@ -1,33 +0,0 @@
-
-#define BORG_NO_PYTHON
-
-#include "../../src/borg/_hashindex.c"
-#include "../../src/borg/cache_sync/cache_sync.c"
-
-#define BUFSZ 32768
-
-int main() {
-    char buf[BUFSZ];
-    int len, ret;
-    CacheSyncCtx *ctx;
-    HashIndex *idx;
-
-    /* capacity, key size, value size */
-    idx = hashindex_init(0, 32, 12);
-    ctx = cache_sync_init(idx);
-
-    while (1) {
-        len = read(0, buf, BUFSZ);
-        if (!len) {
-            break;
-        }
-        ret = cache_sync_feed(ctx, buf, len);
-        if(!ret && cache_sync_error(ctx)) {
-            fprintf(stderr, "error: %s\n", cache_sync_error(ctx));
-            return 1;
-        }
-    }
-    hashindex_free(idx);
-    cache_sync_free(ctx);
-    return 0;
-}

src/borg/cache_sync/cache_sync.c

@@ -1,138 +0,0 @@
-/*
- * Borg cache synchronizer,
- * high level interface.
- *
- * These routines parse msgpacked item metadata and update a HashIndex
- * with all chunks that are referenced from the items.
- *
- * This file only contains some initialization and buffer management.
- *
- * The parser is split in two parts, somewhat similar to lexer/parser combinations:
- *
- * unpack_template.h munches msgpack and calls a specific callback for each object
- * encountered (e.g. beginning of a map, an integer, a string, a map item etc.).
- *
- * unpack.h implements these callbacks and uses another state machine to
- * extract chunk references from it.
- */
-
-#include "unpack.h"
-
-typedef struct {
-    unpack_context ctx;
-
-    char *buf;
-    size_t head;
-    size_t tail;
-    size_t size;
-} CacheSyncCtx;
-
-static CacheSyncCtx *
-cache_sync_init(HashIndex *chunks)
-{
-    CacheSyncCtx *ctx;
-    if (!(ctx = (CacheSyncCtx*)malloc(sizeof(CacheSyncCtx)))) {
-        return NULL;
-    }
-
-    unpack_init(&ctx->ctx);
-    /* needs to be set only once */
-    ctx->ctx.user.chunks = chunks;
-    ctx->ctx.user.totals.size = 0;
-    ctx->ctx.user.totals.num_files = 0;
-    ctx->buf = NULL;
-    ctx->head = 0;
-    ctx->tail = 0;
-    ctx->size = 0;
-
-    return ctx;
-}
-
-static void
-cache_sync_free(CacheSyncCtx *ctx)
-{
-    if(ctx->buf) {
-        free(ctx->buf);
-    }
-    free(ctx);
-}
-
-static const char *
-cache_sync_error(const CacheSyncCtx *ctx)
-{
-    return ctx->ctx.user.last_error;
-}
-
-static uint64_t
-cache_sync_num_files_totals(const CacheSyncCtx *ctx)
-{
-    return ctx->ctx.user.totals.num_files;
-}
-
-static uint64_t
-cache_sync_size_totals(const CacheSyncCtx *ctx)
-{
-    return ctx->ctx.user.totals.size;
-}
-
-/**
- * feed data to the cache synchronizer
- * 0 = abort, 1 = continue
- * abort is a regular condition, check cache_sync_error
- */
-static int
-cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length)
-{
-    size_t new_size;
-    int ret;
-    char *new_buf;
-
-    if(ctx->tail + length > ctx->size) {
-        if((ctx->tail - ctx->head) + length <= ctx->size) {
-            /* |  XXXXX| -> move data in buffer backwards -> |XXXXX  | */
-            memmove(ctx->buf, ctx->buf + ctx->head, ctx->tail - ctx->head);
-            ctx->tail -= ctx->head;
-            ctx->head = 0;
-        } else {
-            /* must expand buffer to fit all data */
-            new_size = (ctx->tail - ctx->head) + length;
-            new_buf = (char*) malloc(new_size);
-            if(!new_buf) {
-                ctx->ctx.user.last_error = "cache_sync_feed: unable to allocate buffer";
-                return 0;
-            }
-            if(ctx->buf) {
-                memcpy(new_buf, ctx->buf + ctx->head, ctx->tail - ctx->head);
-                free(ctx->buf);
-            }
-            ctx->buf = new_buf;
-            ctx->tail -= ctx->head;
-            ctx->head = 0;
-            ctx->size = new_size;
-        }
-    }
-
-    memcpy(ctx->buf + ctx->tail, data, length);
-    ctx->tail += length;
-
-    while(1) {
-        if(ctx->head >= ctx->tail) {
-            return 1;  /* request more bytes */
-        }
-
-        ret = unpack_execute(&ctx->ctx, ctx->buf, ctx->tail, &ctx->head);
-        if(ret == 1) {
-            unpack_init(&ctx->ctx);
-            continue;
-        } else if(ret == 0) {
-            return 1;
-        } else {
-            if(!ctx->ctx.user.last_error) {
-                ctx->ctx.user.last_error = "Unknown error";
-            }
-            return 0;
-        }
-    }
-    /* unreachable */
-    return 1;
-}

src/borg/cache_sync/sysdep.h

@@ -1,194 +0,0 @@
-/*
- * MessagePack system dependencies
- *
- * Copyright (C) 2008-2010 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        https://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-#ifndef MSGPACK_SYSDEP_H__
-#define MSGPACK_SYSDEP_H__
-
-#include <stdlib.h>
-#include <stddef.h>
-#if defined(_MSC_VER) && _MSC_VER < 1600
-typedef __int8 int8_t;
-typedef unsigned __int8 uint8_t;
-typedef __int16 int16_t;
-typedef unsigned __int16 uint16_t;
-typedef __int32 int32_t;
-typedef unsigned __int32 uint32_t;
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-#elif defined(_MSC_VER)  // && _MSC_VER >= 1600
-#include <stdint.h>
-#else
-#include <stdint.h>
-#include <stdbool.h>
-#endif
-
-#ifdef _WIN32
-#define _msgpack_atomic_counter_header <windows.h>
-typedef long _msgpack_atomic_counter_t;
-#define _msgpack_sync_decr_and_fetch(ptr) InterlockedDecrement(ptr)
-#define _msgpack_sync_incr_and_fetch(ptr) InterlockedIncrement(ptr)
-#elif defined(__GNUC__) && ((__GNUC__*10 + __GNUC_MINOR__) < 41)
-#define _msgpack_atomic_counter_header "gcc_atomic.h"
-#else
-typedef unsigned int _msgpack_atomic_counter_t;
-#define _msgpack_sync_decr_and_fetch(ptr) __sync_sub_and_fetch(ptr, 1)
-#define _msgpack_sync_incr_and_fetch(ptr) __sync_add_and_fetch(ptr, 1)
-#endif
-
-#ifdef _WIN32
-
-#ifdef __cplusplus
-/* numeric_limits<T>::min,max */
-#ifdef max
-#undef max
-#endif
-#ifdef min
-#undef min
-#endif
-#endif
-
-#else
-#include <arpa/inet.h>  /* __BYTE_ORDER */
-#endif
-
-#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-#define __LITTLE_ENDIAN__
-#elif __BYTE_ORDER == __BIG_ENDIAN
-#define __BIG_ENDIAN__
-#elif _WIN32
-#define __LITTLE_ENDIAN__
-#endif
-#endif
-
-
-#ifdef __LITTLE_ENDIAN__
-
-#ifdef _WIN32
-#  if defined(ntohs)
-#    define _msgpack_be16(x) ntohs(x)
-#  elif defined(_byteswap_ushort) || (defined(_MSC_VER) && _MSC_VER >= 1400)
-#    define _msgpack_be16(x) ((uint16_t)_byteswap_ushort((unsigned short)x))
-#  else
-#    define _msgpack_be16(x) ( \
-        ((((uint16_t)x) <<  8) ) | \
-        ((((uint16_t)x) >>  8) ) )
-#  endif
-#else
-#  define _msgpack_be16(x) ntohs(x)
-#endif
-
-#ifdef _WIN32
-#  if defined(ntohl)
-#    define _msgpack_be32(x) ntohl(x)
-#  elif defined(_byteswap_ulong) || (defined(_MSC_VER) && _MSC_VER >= 1400)
-#    define _msgpack_be32(x) ((uint32_t)_byteswap_ulong((unsigned long)x))
-#  else
-#    define _msgpack_be32(x) \
-        ( ((((uint32_t)x) << 24)               ) | \
-          ((((uint32_t)x) <<  8) & 0x00ff0000U ) | \
-          ((((uint32_t)x) >>  8) & 0x0000ff00U ) | \
-          ((((uint32_t)x) >> 24)               ) )
-#  endif
-#else
-#  define _msgpack_be32(x) ntohl(x)
-#endif
-
-#if defined(_byteswap_uint64) || (defined(_MSC_VER) && _MSC_VER >= 1400)
-#  define _msgpack_be64(x) (_byteswap_uint64(x))
-#elif defined(bswap_64)
-#  define _msgpack_be64(x) bswap_64(x)
-#elif defined(__DARWIN_OSSwapInt64)
-#  define _msgpack_be64(x) __DARWIN_OSSwapInt64(x)
-#else
-#define _msgpack_be64(x) \
-    ( ((((uint64_t)x) << 56)                         ) | \
-      ((((uint64_t)x) << 40) & 0x00ff000000000000ULL ) | \
-      ((((uint64_t)x) << 24) & 0x0000ff0000000000ULL ) | \
-      ((((uint64_t)x) <<  8) & 0x000000ff00000000ULL ) | \
-      ((((uint64_t)x) >>  8) & 0x00000000ff000000ULL ) | \
-      ((((uint64_t)x) >> 24) & 0x0000000000ff0000ULL ) | \
-      ((((uint64_t)x) >> 40) & 0x000000000000ff00ULL ) | \
-      ((((uint64_t)x) >> 56)                         ) )
-#endif
-
-#define _msgpack_load16(cast, from) ((cast)( \
-        (((uint16_t)((uint8_t*)(from))[0]) << 8) | \
-        (((uint16_t)((uint8_t*)(from))[1])     ) ))
-
-#define _msgpack_load32(cast, from) ((cast)( \
-        (((uint32_t)((uint8_t*)(from))[0]) << 24) | \
-        (((uint32_t)((uint8_t*)(from))[1]) << 16) | \
-        (((uint32_t)((uint8_t*)(from))[2]) <<  8) | \
-        (((uint32_t)((uint8_t*)(from))[3])      ) ))
-
-#define _msgpack_load64(cast, from) ((cast)( \
-        (((uint64_t)((uint8_t*)(from))[0]) << 56) | \
-        (((uint64_t)((uint8_t*)(from))[1]) << 48) | \
-        (((uint64_t)((uint8_t*)(from))[2]) << 40) | \
-        (((uint64_t)((uint8_t*)(from))[3]) << 32) | \
-        (((uint64_t)((uint8_t*)(from))[4]) << 24) | \
-        (((uint64_t)((uint8_t*)(from))[5]) << 16) | \
-        (((uint64_t)((uint8_t*)(from))[6]) << 8)  | \
-        (((uint64_t)((uint8_t*)(from))[7])     )  ))
-
-#else
-
-#define _msgpack_be16(x) (x)
-#define _msgpack_be32(x) (x)
-#define _msgpack_be64(x) (x)
-
-#define _msgpack_load16(cast, from) ((cast)( \
-        (((uint16_t)((uint8_t*)from)[0]) << 8) | \
-        (((uint16_t)((uint8_t*)from)[1])     ) ))
-
-#define _msgpack_load32(cast, from) ((cast)( \
-        (((uint32_t)((uint8_t*)from)[0]) << 24) | \
-        (((uint32_t)((uint8_t*)from)[1]) << 16) | \
-        (((uint32_t)((uint8_t*)from)[2]) <<  8) | \
-        (((uint32_t)((uint8_t*)from)[3])      ) ))
-
-#define _msgpack_load64(cast, from) ((cast)( \
-        (((uint64_t)((uint8_t*)from)[0]) << 56) | \
-        (((uint64_t)((uint8_t*)from)[1]) << 48) | \
-        (((uint64_t)((uint8_t*)from)[2]) << 40) | \
-        (((uint64_t)((uint8_t*)from)[3]) << 32) | \
-        (((uint64_t)((uint8_t*)from)[4]) << 24) | \
-        (((uint64_t)((uint8_t*)from)[5]) << 16) | \
-        (((uint64_t)((uint8_t*)from)[6]) << 8)  | \
-        (((uint64_t)((uint8_t*)from)[7])     )  ))
-#endif
-
-
-#define _msgpack_store16(to, num) \
-    do { uint16_t val = _msgpack_be16(num); memcpy(to, &val, 2); } while(0)
-#define _msgpack_store32(to, num) \
-    do { uint32_t val = _msgpack_be32(num); memcpy(to, &val, 4); } while(0)
-#define _msgpack_store64(to, num) \
-    do { uint64_t val = _msgpack_be64(num); memcpy(to, &val, 8); } while(0)
-
-/*
-#define _msgpack_load16(cast, from) \
-    ({ cast val; memcpy(&val, (char*)from, 2); _msgpack_be16(val); })
-#define _msgpack_load32(cast, from) \
-    ({ cast val; memcpy(&val, (char*)from, 4); _msgpack_be32(val); })
-#define _msgpack_load64(cast, from) \
-    ({ cast val; memcpy(&val, (char*)from, 8); _msgpack_be64(val); })
-*/
-
-
-#endif /* msgpack/sysdep.h */

src/borg/cache_sync/unpack.h

@@ -1,416 +0,0 @@
-/*
- * Borg cache synchronizer,
- * based on a MessagePack for Python unpacking routine
- *
- * Copyright (C) 2009 Naoki INADA
- * Copyright (c) 2017 Marian Beermann
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        https://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-
-/*
- * This limits the depth of the structures we can unpack, i.e. how many containers
- * are nestable.
- */
-#define MSGPACK_EMBED_STACK_SIZE  (16)
-#include "unpack_define.h"
-
-// 2**32 - 1025
-#define _MAX_VALUE ( (uint32_t) 4294966271UL )
-
-#define MIN(x, y) ((x) < (y) ? (x): (y))
-
-#ifdef DEBUG
-#define SET_LAST_ERROR(msg) \
-  fprintf(stderr, "cache_sync parse error: %s\n", (msg)); \
-  u->last_error = (msg);
-#else
-#define SET_LAST_ERROR(msg) \
-  u->last_error = (msg);
-#endif
-
-typedef struct unpack_user {
-    /* Item.chunks is at the top level; we don't care about anything else,
-     * only need to track the current level to navigate arbitrary and unknown structure.
-     * To discern keys from everything else on the top level we use expect_map_item_end.
-     */
-    int level;
-
-    const char *last_error;
-
-    HashIndex *chunks;
-
-    /*
-     * We don't care about most stuff. This flag tells us whether we're at the chunks structure,
-     * meaning:
-     * {'foo': 'bar', 'chunks': [...], 'stuff': ... }
-     *                        ^-HERE-^
-     */
-    int inside_chunks;
-
-    /* does this item have a chunks list in it? */
-    int has_chunks;
-
-    enum {
-        /* the next thing is a map key at the Item root level,
-         * and it might be e.g. the "chunks" key we're looking for */
-        expect_map_key,
-
-        /* blocking state to expect_map_key
-         * {     'stuff': <complex and arbitrary structure>,     'chunks': [
-         * emk     ->   emie    ->   ->       ->      ->   emk   ecb       eeboce
-         *                (nested containers are tracked via level)
-         * emk=expect_map_key, emie=expect_map_item_end, ecb=expect_chunks_begin,
-         * eeboce=expect_entry_begin_or_chunks_end
-         */
-        expect_map_item_end,
-
-        /* next thing must be the chunks array (array) */
-        expect_chunks_begin,
-
-        /* next thing must either be another CLE (array) or end of Item.chunks (array_end) */
-        expect_entry_begin_or_chunks_end,
-
-        /*
-         * processing ChunkListEntry tuple:
-         * expect_key, expect_size, expect_entry_end
-         */
-        /* next thing must be the key (raw, l=32) */
-        expect_key,
-        /* next thing must be the size (int) */
-        expect_size,
-        /* next thing must be the end of the CLE (array_end) */
-        expect_entry_end,
-
-        expect_item_begin
-    } expect;
-
-    /* collect values here for current chunklist entry */
-    struct {
-        unsigned char key[32];
-        uint32_t size;
-    } current;
-
-    /* summing up chunks sizes here within a single item */
-    struct {
-        uint64_t size;
-    } item;
-
-    /* total sizes and files count coming from all files */
-    struct {
-        uint64_t size, num_files;
-    } totals;
-
-} unpack_user;
-
-struct unpack_context;
-typedef struct unpack_context unpack_context;
-typedef int (*execute_fn)(unpack_context *ctx, const char* data, size_t len, size_t* off);
-
-#define UNEXPECTED(what)                                            \
-    if(u->inside_chunks || u->expect == expect_map_key) { \
-        SET_LAST_ERROR("Unexpected object: " what);                 \
-        return -1;                                                  \
-    }
-
-static inline void unpack_init_user_state(unpack_user *u)
-{
-    u->last_error = NULL;
-    u->level = 0;
-    u->inside_chunks = 0;
-    u->expect = expect_item_begin;
-}
-
-static inline int unpack_callback_uint64(unpack_user* u, int64_t d)
-{
-    switch(u->expect) {
-        case expect_size:
-            u->current.size = d;
-            u->expect = expect_entry_end;
-            break;
-        default:
-            UNEXPECTED("integer");
-    }
-    return 0;
-}
-
-static inline int unpack_callback_uint32(unpack_user* u, uint32_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_uint16(unpack_user* u, uint16_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_uint8(unpack_user* u, uint8_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_int64(unpack_user* u, uint64_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_int32(unpack_user* u, int32_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_int16(unpack_user* u, int16_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-static inline int unpack_callback_int8(unpack_user* u, int8_t d)
-{
-    return unpack_callback_uint64(u, d);
-}
-
-/* Ain't got anything to do with those floats */
-static inline int unpack_callback_double(unpack_user* u, double d)
-{
-    (void)d;
-    UNEXPECTED("double");
-    return 0;
-}
-
-static inline int unpack_callback_float(unpack_user* u, float d)
-{
-    (void)d;
-    UNEXPECTED("float");
-    return 0;
-}
-
-/* nil/true/false — I/don't/care */
-static inline int unpack_callback_nil(unpack_user* u)
-{
-    UNEXPECTED("nil");
-    return 0;
-}
-
-static inline int unpack_callback_true(unpack_user* u)
-{
-    UNEXPECTED("true");
-    return 0;
-}
-
-static inline int unpack_callback_false(unpack_user* u)
-{
-    UNEXPECTED("false");
-    return 0;
-}
-
-static inline int unpack_callback_array(unpack_user* u, unsigned int n)
-{
-    switch(u->expect) {
-    case expect_chunks_begin:
-        /* b'chunks': [
-         *            ^ */
-        u->expect = expect_entry_begin_or_chunks_end;
-        break;
-    case expect_entry_begin_or_chunks_end:
-        /* b'chunks': [ (
-         *              ^ */
-        if(n != 2) {
-            SET_LAST_ERROR("Invalid chunk list entry length");
-            return -1;
-        }
-        u->expect = expect_key;
-        break;
-    default:
-        if(u->inside_chunks) {
-            SET_LAST_ERROR("Unexpected array start");
-            return -1;
-        } else {
-            u->level++;
-            return 0;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_array_item(unpack_user* u, unsigned int current)
-{
-    (void)u; (void)current;
-    return 0;
-}
-
-static inline int unpack_callback_array_end(unpack_user* u)
-{
-    uint32_t *cache_entry;
-    uint32_t cache_values[3];
-    uint64_t refcount;
-
-    switch(u->expect) {
-    case expect_entry_end:
-        /* b'chunks': [ ( b'1234...', 123, 345 )
-         *                                     ^ */
-        cache_entry = (uint32_t*) hashindex_get(u->chunks, u->current.key);
-        if(cache_entry) {
-            refcount = _le32toh(cache_entry[0]);
-            if(refcount > _MAX_VALUE) {
-                SET_LAST_ERROR("invalid reference count");
-                return -1;
-            }
-            refcount += 1;
-            cache_entry[0] = _htole32(MIN(refcount, _MAX_VALUE));
-        } else {
-            /* refcount, size */
-            cache_values[0] = _htole32(1);
-            cache_values[1] = _htole32(u->current.size);
-            if(!hashindex_set(u->chunks, u->current.key, cache_values)) {
-                SET_LAST_ERROR("hashindex_set failed");
-                return -1;
-            }
-        }
-        u->item.size += u->current.size;
-        u->expect = expect_entry_begin_or_chunks_end;
-        break;
-    case expect_entry_begin_or_chunks_end:
-        /* b'chunks': [ ]
-         *              ^ */
-        /* end of Item.chunks */
-        u->inside_chunks = 0;
-        u->expect = expect_map_item_end;
-        break;
-    default:
-        if(u->inside_chunks) {
-            SET_LAST_ERROR("Invalid state transition (unexpected array end)");
-            return -1;
-        } else {
-            u->level--;
-            return 0;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_map(unpack_user* u, unsigned int n)
-{
-    (void)n;
-
-    if(u->level == 0) {
-        if(u->expect != expect_item_begin) {
-            SET_LAST_ERROR("Invalid state transition");  /* unreachable */
-            return -1;
-        }
-        /* This begins a new Item */
-        u->expect = expect_map_key;
-        u->has_chunks = 0;
-        u->item.size = 0;
-    }
-
-    if(u->inside_chunks) {
-        UNEXPECTED("map");
-    }
-
-    u->level++;
-
-    return 0;
-}
-
-static inline int unpack_callback_map_item(unpack_user* u, unsigned int current)
-{
-    (void)u; (void)current;
-
-    if(u->level == 1) {
-        switch(u->expect) {
-        case expect_map_item_end:
-            u->expect = expect_map_key;
-            break;
-        default:
-            SET_LAST_ERROR("Unexpected map item");
-            return -1;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_map_end(unpack_user* u)
-{
-    u->level--;
-    if(u->inside_chunks) {
-        SET_LAST_ERROR("Unexpected map end");
-        return -1;
-    }
-    if(u->level == 0) {
-        /* This ends processing of an Item */
-        if(u->has_chunks) {
-            u->totals.num_files += 1;
-            u->totals.size += u->item.size;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_raw(unpack_user* u, const char* b, const char* p, unsigned int length)
-{
-    /* raw = what Borg uses for text stuff */
-    /* Note: p points to an internal buffer which contains l bytes. */
-    (void)b;
-
-    switch(u->expect) {
-    case expect_map_key:
-        if(length == 6 && !memcmp("chunks", p, 6)) {
-            u->expect = expect_chunks_begin;
-            u->inside_chunks = 1;
-            u->has_chunks = 1;
-        } else {
-            u->expect = expect_map_item_end;
-        }
-        break;
-    default:
-        if(u->inside_chunks) {
-            SET_LAST_ERROR("Unexpected raw in chunks structure");
-            return -1;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_bin(unpack_user* u, const char* b, const char* p, unsigned int length)
-{
-    /* bin = what Borg uses for binary stuff */
-    /* Note: p points to an internal buffer which contains l bytes. */
-    (void)b;
-
-    switch(u->expect) {
-    case expect_key:
-        if(length != 32) {
-            SET_LAST_ERROR("Incorrect key length");
-            return -1;
-        }
-        memcpy(u->current.key, p, 32);
-        u->expect = expect_size;
-        break;
-    default:
-        if(u->inside_chunks) {
-            SET_LAST_ERROR("Unexpected bytes in chunks structure");
-            return -1;
-        }
-    }
-    return 0;
-}
-
-static inline int unpack_callback_ext(unpack_user* u, const char* base, const char* pos,
-                                      unsigned int length)
-{
-    (void)u; (void)base; (void)pos; (void)length;
-    UNEXPECTED("ext");
-    return 0;
-}
-
-#include "unpack_template.h"

src/borg/cache_sync/unpack_define.h

@@ -1,95 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2010 FURUHASHI Sadayuki
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        https://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- */
-#ifndef MSGPACK_UNPACK_DEFINE_H__
-#define MSGPACK_UNPACK_DEFINE_H__
-
-#include "sysdep.h"
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <stdio.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-#ifndef MSGPACK_EMBED_STACK_SIZE
-#define MSGPACK_EMBED_STACK_SIZE 32
-#endif
-
-
-// CS is first byte & 0x1f
-typedef enum {
-    CS_HEADER            = 0x00,  // nil
-
-    //CS_                = 0x01,
-    //CS_                = 0x02,  // false
-    //CS_                = 0x03,  // true
-
-    CS_BIN_8             = 0x04,
-    CS_BIN_16            = 0x05,
-    CS_BIN_32            = 0x06,
-
-    CS_EXT_8             = 0x07,
-    CS_EXT_16            = 0x08,
-    CS_EXT_32            = 0x09,
-
-    CS_FLOAT             = 0x0a,
-    CS_DOUBLE            = 0x0b,
-    CS_UINT_8            = 0x0c,
-    CS_UINT_16           = 0x0d,
-    CS_UINT_32           = 0x0e,
-    CS_UINT_64           = 0x0f,
-    CS_INT_8             = 0x10,
-    CS_INT_16            = 0x11,
-    CS_INT_32            = 0x12,
-    CS_INT_64            = 0x13,
-
-    //CS_FIXEXT1           = 0x14,
-    //CS_FIXEXT2           = 0x15,
-    //CS_FIXEXT4           = 0x16,
-    //CS_FIXEXT8           = 0x17,
-    //CS_FIXEXT16          = 0x18,
-
-    CS_RAW_8             = 0x19,
-    CS_RAW_16            = 0x1a,
-    CS_RAW_32            = 0x1b,
-    CS_ARRAY_16          = 0x1c,
-    CS_ARRAY_32          = 0x1d,
-    CS_MAP_16            = 0x1e,
-    CS_MAP_32            = 0x1f,
-
-    ACS_RAW_VALUE,
-    ACS_BIN_VALUE,
-    ACS_EXT_VALUE,
-} msgpack_unpack_state;
-
-
-typedef enum {
-    CT_ARRAY_ITEM,
-    CT_MAP_KEY,
-    CT_MAP_VALUE,
-} msgpack_container_type;
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* msgpack/unpack_define.h */

src/borg/cache_sync/unpack_template.h

@@ -1,365 +0,0 @@
-/*
- * MessagePack unpacking routine template
- *
- * Copyright (C) 2008-2010 FURUHASHI Sadayuki
- * Copyright (c) 2017 Marian Beermann
- *
- *    Licensed under the Apache License, Version 2.0 (the "License");
- *    you may not use this file except in compliance with the License.
- *    You may obtain a copy of the License at
- *
- *        https://www.apache.org/licenses/LICENSE-2.0
- *
- *    Unless required by applicable law or agreed to in writing, software
- *    distributed under the License is distributed on an "AS IS" BASIS,
- *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *    See the License for the specific language governing permissions and
- *    limitations under the License.
- *
- *
- * This has been slightly adapted from the vanilla msgpack-{c, python} version.
- * Since cache_sync does not intend to build an output data structure,
- * msgpack_unpack_object and all of its uses was removed.
- */
-
-#ifndef USE_CASE_RANGE
-#if !defined(_MSC_VER)
-#define USE_CASE_RANGE
-#endif
-#endif
-
-typedef struct unpack_stack {
-    size_t size;
-    size_t count;
-    unsigned int ct;
-} unpack_stack;
-
-struct unpack_context {
-    unpack_user user;
-    unsigned int cs;
-    unsigned int trail;
-    unsigned int top;
-    unpack_stack stack[MSGPACK_EMBED_STACK_SIZE];
-};
-
-static inline void unpack_init(unpack_context* ctx)
-{
-    ctx->cs = CS_HEADER;
-    ctx->trail = 0;
-    ctx->top = 0;
-    unpack_init_user_state(&ctx->user);
-}
-
-#define construct 1
-
-static inline int unpack_execute(unpack_context* ctx, const char* data, size_t len, size_t* off)
-{
-    const unsigned char* p = (unsigned char*)data + *off;
-    const unsigned char* const pe = (unsigned char*)data + len;
-    const void* n = NULL;
-
-    unsigned int trail = ctx->trail;
-    unsigned int cs = ctx->cs;
-    unsigned int top = ctx->top;
-    unpack_stack* stack = ctx->stack;
-    unpack_user* user = &ctx->user;
-
-    unpack_stack* c = NULL;
-
-    int ret;
-
-    assert(len >= *off);
-
-#define construct_cb(name) \
-    construct && unpack_callback ## name
-
-#define push_simple_value(func) \
-    if(construct_cb(func)(user) < 0) { goto _failed; } \
-    goto _push
-#define push_fixed_value(func, arg) \
-    if(construct_cb(func)(user, arg) < 0) { goto _failed; } \
-    goto _push
-#define push_variable_value(func, base, pos, len) \
-    if(construct_cb(func)(user, \
-        (const char*)base, (const char*)pos, len) < 0) { goto _failed; } \
-    goto _push
-
-#define again_fixed_trail(_cs, trail_len) \
-    trail = trail_len; \
-    cs = _cs; \
-    goto _fixed_trail_again
-#define again_fixed_trail_if_zero(_cs, trail_len, ifzero) \
-    trail = trail_len; \
-    if(trail == 0) { goto ifzero; } \
-    cs = _cs; \
-    goto _fixed_trail_again
-
-#define start_container(func, count_, ct_) \
-    if(top >= MSGPACK_EMBED_STACK_SIZE) { goto _failed; } /* FIXME */ \
-    if(construct_cb(func)(user, count_) < 0) { goto _failed; } \
-    if((count_) == 0) { \
-        if (construct_cb(func##_end)(user) < 0) { goto _failed; } \
-        goto _push; } \
-    stack[top].ct = ct_; \
-    stack[top].size  = count_; \
-    stack[top].count = 0; \
-    ++top; \
-    goto _header_again
-
-#define NEXT_CS(p)  ((unsigned int)*p & 0x1f)
-
-#ifdef USE_CASE_RANGE
-#define SWITCH_RANGE_BEGIN     switch(*p) {
-#define SWITCH_RANGE(FROM, TO) case FROM ... TO:
-#define SWITCH_RANGE_DEFAULT   default:
-#define SWITCH_RANGE_END       }
-#else
-#define SWITCH_RANGE_BEGIN     { if(0) {
-#define SWITCH_RANGE(FROM, TO) } else if(FROM <= *p && *p <= TO) {
-#define SWITCH_RANGE_DEFAULT   } else {
-#define SWITCH_RANGE_END       } }
-#endif
-
-    if(p == pe) { goto _out; }
-    do {
-        switch(cs) {
-        case CS_HEADER:
-            SWITCH_RANGE_BEGIN
-            SWITCH_RANGE(0x00, 0x7f)  // Positive Fixnum
-                push_fixed_value(_uint8, *(uint8_t*)p);
-            SWITCH_RANGE(0xe0, 0xff)  // Negative Fixnum
-                push_fixed_value(_int8, *(int8_t*)p);
-            SWITCH_RANGE(0xc0, 0xdf)  // Variable
-                switch(*p) {
-                case 0xc0:  // nil
-                    push_simple_value(_nil);
-                //case 0xc1:  // never used
-                case 0xc2:  // false
-                    push_simple_value(_false);
-                case 0xc3:  // true
-                    push_simple_value(_true);
-                case 0xc4:  // bin 8
-                    again_fixed_trail(NEXT_CS(p), 1);
-                case 0xc5:  // bin 16
-                    again_fixed_trail(NEXT_CS(p), 2);
-                case 0xc6:  // bin 32
-                    again_fixed_trail(NEXT_CS(p), 4);
-                case 0xc7:  // ext 8
-                    again_fixed_trail(NEXT_CS(p), 1);
-                case 0xc8:  // ext 16
-                    again_fixed_trail(NEXT_CS(p), 2);
-                case 0xc9:  // ext 32
-                    again_fixed_trail(NEXT_CS(p), 4);
-                case 0xca:  // float
-                case 0xcb:  // double
-                case 0xcc:  // unsigned int  8
-                case 0xcd:  // unsigned int 16
-                case 0xce:  // unsigned int 32
-                case 0xcf:  // unsigned int 64
-                case 0xd0:  // signed int  8
-                case 0xd1:  // signed int 16
-                case 0xd2:  // signed int 32
-                case 0xd3:  // signed int 64
-                    again_fixed_trail(NEXT_CS(p), 1 << (((unsigned int)*p) & 0x03));
-                case 0xd4:  // fixext 1
-                case 0xd5:  // fixext 2
-                case 0xd6:  // fixext 4
-                case 0xd7:  // fixext 8
-                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 
-                                              (1 << (((unsigned int)*p) & 0x03))+1,
-                                              _ext_zero);
-                case 0xd8:  // fixext 16
-                    again_fixed_trail_if_zero(ACS_EXT_VALUE, 16+1, _ext_zero);
-                case 0xd9:  // str 8
-                    again_fixed_trail(NEXT_CS(p), 1);
-                case 0xda:  // raw 16
-                case 0xdb:  // raw 32
-                case 0xdc:  // array 16
-                case 0xdd:  // array 32
-                case 0xde:  // map 16
-                case 0xdf:  // map 32
-                    again_fixed_trail(NEXT_CS(p), 2 << (((unsigned int)*p) & 0x01));
-                default:
-                    goto _failed;
-                }
-            SWITCH_RANGE(0xa0, 0xbf)  // FixRaw
-                again_fixed_trail_if_zero(ACS_RAW_VALUE, ((unsigned int)*p & 0x1f), _raw_zero);
-            SWITCH_RANGE(0x90, 0x9f)  // FixArray
-                start_container(_array, ((unsigned int)*p) & 0x0f, CT_ARRAY_ITEM);
-            SWITCH_RANGE(0x80, 0x8f)  // FixMap
-                start_container(_map, ((unsigned int)*p) & 0x0f, CT_MAP_KEY);
-
-            SWITCH_RANGE_DEFAULT
-                goto _failed;
-            SWITCH_RANGE_END
-            // end CS_HEADER
-
-
-        _fixed_trail_again:
-            ++p; // fallthrough
-
-        default:
-            if((size_t)(pe - p) < trail) { goto _out; }
-            n = p;  p += trail - 1;
-            switch(cs) {
-            case CS_EXT_8:
-                again_fixed_trail_if_zero(ACS_EXT_VALUE, *(uint8_t*)n+1, _ext_zero);
-            case CS_EXT_16:
-                again_fixed_trail_if_zero(ACS_EXT_VALUE,
-                                          _msgpack_load16(uint16_t,n)+1,
-                                          _ext_zero);
-            case CS_EXT_32:
-                again_fixed_trail_if_zero(ACS_EXT_VALUE,
-                                          _msgpack_load32(uint32_t,n)+1,
-                                          _ext_zero);
-            case CS_FLOAT: {
-                    union { uint32_t i; float f; } mem;
-                    mem.i = _msgpack_load32(uint32_t,n);
-                    push_fixed_value(_float, mem.f); }
-            case CS_DOUBLE: {
-                    union { uint64_t i; double f; } mem;
-                    mem.i = _msgpack_load64(uint64_t,n);
-#if defined(__arm__) && !(__ARM_EABI__) // arm-oabi
-                    // https://github.com/msgpack/msgpack-perl/pull/1
-                    mem.i = (mem.i & 0xFFFFFFFFUL) << 32UL | (mem.i >> 32UL);
-#endif
-                    push_fixed_value(_double, mem.f); }
-            case CS_UINT_8:
-                push_fixed_value(_uint8, *(uint8_t*)n);
-            case CS_UINT_16:
-                push_fixed_value(_uint16, _msgpack_load16(uint16_t,n));
-            case CS_UINT_32:
-                push_fixed_value(_uint32, _msgpack_load32(uint32_t,n));
-            case CS_UINT_64:
-                push_fixed_value(_uint64, _msgpack_load64(uint64_t,n));
-
-            case CS_INT_8:
-                push_fixed_value(_int8, *(int8_t*)n);
-            case CS_INT_16:
-                push_fixed_value(_int16, _msgpack_load16(int16_t,n));
-            case CS_INT_32:
-                push_fixed_value(_int32, _msgpack_load32(int32_t,n));
-            case CS_INT_64:
-                push_fixed_value(_int64, _msgpack_load64(int64_t,n));
-
-            case CS_BIN_8:
-                again_fixed_trail_if_zero(ACS_BIN_VALUE, *(uint8_t*)n, _bin_zero);
-            case CS_BIN_16:
-                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load16(uint16_t,n), _bin_zero);
-            case CS_BIN_32:
-                again_fixed_trail_if_zero(ACS_BIN_VALUE, _msgpack_load32(uint32_t,n), _bin_zero);
-            case ACS_BIN_VALUE:
-            _bin_zero:
-                push_variable_value(_bin, data, n, trail);
-
-            case CS_RAW_8:
-                again_fixed_trail_if_zero(ACS_RAW_VALUE, *(uint8_t*)n, _raw_zero);
-            case CS_RAW_16:
-                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load16(uint16_t,n), _raw_zero);
-            case CS_RAW_32:
-                again_fixed_trail_if_zero(ACS_RAW_VALUE, _msgpack_load32(uint32_t,n), _raw_zero);
-            case ACS_RAW_VALUE:
-            _raw_zero:
-                push_variable_value(_raw, data, n, trail);
-
-            case ACS_EXT_VALUE:
-            _ext_zero:
-                push_variable_value(_ext, data, n, trail);
-
-            case CS_ARRAY_16:
-                start_container(_array, _msgpack_load16(uint16_t,n), CT_ARRAY_ITEM);
-            case CS_ARRAY_32:
-                /* FIXME security guard */
-                start_container(_array, _msgpack_load32(uint32_t,n), CT_ARRAY_ITEM);
-
-            case CS_MAP_16:
-                start_container(_map, _msgpack_load16(uint16_t,n), CT_MAP_KEY);
-            case CS_MAP_32:
-                /* FIXME security guard */
-                start_container(_map, _msgpack_load32(uint32_t,n), CT_MAP_KEY);
-
-            default:
-                goto _failed;
-            }
-        }
-
-_push:
-    if(top == 0) { goto _finish; }
-    c = &stack[top-1];
-    switch(c->ct) {
-    case CT_ARRAY_ITEM:
-        if(construct_cb(_array_item)(user, c->count) < 0) { goto _failed; }
-        if(++c->count == c->size) {
-            if (construct_cb(_array_end)(user) < 0) { goto _failed; }
-            --top;
-            /*printf("stack pop %d\n", top);*/
-            goto _push;
-        }
-        goto _header_again;
-    case CT_MAP_KEY:
-        c->ct = CT_MAP_VALUE;
-        goto _header_again;
-    case CT_MAP_VALUE:
-        if(construct_cb(_map_item)(user, c->count) < 0) { goto _failed; }
-        if(++c->count == c->size) {
-            if (construct_cb(_map_end)(user) < 0) { goto _failed; }
-            --top;
-            /*printf("stack pop %d\n", top);*/
-            goto _push;
-        }
-        c->ct = CT_MAP_KEY;
-        goto _header_again;
-
-    default:
-        goto _failed;
-    }
-
-_header_again:
-        cs = CS_HEADER;
-        ++p;
-    } while(p != pe);
-    goto _out;
-
-
-_finish:
-    if (!construct)
-        unpack_callback_nil(user);
-    ++p;
-    ret = 1;
-    /* printf("-- finish --\n"); */
-    goto _end;
-
-_failed:
-    /* printf("** FAILED **\n"); */
-    ret = -1;
-    goto _end;
-
-_out:
-    ret = 0;
-    goto _end;
-
-_end:
-    ctx->cs = cs;
-    ctx->trail = trail;
-    ctx->top = top;
-    *off = p - (const unsigned char*)data;
-
-    return ret;
-#undef construct_cb
-}
-
-#undef SWITCH_RANGE_BEGIN
-#undef SWITCH_RANGE
-#undef SWITCH_RANGE_DEFAULT
-#undef SWITCH_RANGE_END
-#undef push_simple_value
-#undef push_fixed_value
-#undef push_variable_value
-#undef again_fixed_trail
-#undef again_fixed_trail_if_zero
-#undef start_container
-#undef construct
-
-#undef NEXT_CS
-
-/* vim: set ts=4 sw=4 sts=4 expandtab  */

src/borg/hashindex.pyx

@@ -34,19 +34,7 @@ cdef extern from "_hashindex.c":
     double HASH_MAX_LOAD
 
 
-cdef extern from "cache_sync/cache_sync.c":
-    ctypedef struct CacheSyncCtx:
-        pass
-
-    CacheSyncCtx *cache_sync_init(HashIndex *chunks)
-    const char *cache_sync_error(const CacheSyncCtx *ctx)
-    uint64_t cache_sync_num_files_totals(const CacheSyncCtx *ctx)
-    uint64_t cache_sync_size_totals(const CacheSyncCtx *ctx)
-    int cache_sync_feed(CacheSyncCtx *ctx, void *data, uint32_t length)
-    void cache_sync_free(CacheSyncCtx *ctx)
-
-    uint32_t _MAX_VALUE
-
+_MAX_VALUE = 4294966271UL  # 2**32 - 1025
 
 cdef _NoDefault = object()
 
@@ -592,42 +580,3 @@ cdef class ChunkKeyIterator:
         cdef uint32_t refcount = _le32toh(value[0])
         assert refcount <= _MAX_VALUE, "invalid reference count"
         return (<char *>self.key)[:self.key_size], ChunkIndexEntry(refcount, _le32toh(value[1]))
-
-
-cdef Py_buffer ro_buffer(object data) except *:
-    cdef Py_buffer view
-    PyObject_GetBuffer(data, &view, PyBUF_SIMPLE)
-    return view
-
-
-cdef class CacheSynchronizer:
-    cdef ChunkIndex chunks
-    cdef CacheSyncCtx *sync
-
-    def __cinit__(self, chunks):
-        self.chunks = chunks
-        self.sync = cache_sync_init(self.chunks.index)
-        if not self.sync:
-            raise Exception('cache_sync_init failed')
-
-    def __dealloc__(self):
-        if self.sync:
-            cache_sync_free(self.sync)
-
-    def feed(self, chunk):
-        cdef Py_buffer chunk_buf = ro_buffer(chunk)
-        cdef int rc
-        rc = cache_sync_feed(self.sync, chunk_buf.buf, chunk_buf.len)
-        PyBuffer_Release(&chunk_buf)
-        if not rc:
-            error = cache_sync_error(self.sync)
-            if error != NULL:
-                raise ValueError('cache_sync_feed failed: ' + error.decode('ascii'))
-
-    @property
-    def num_files_totals(self):
-        return cache_sync_num_files_totals(self.sync)
-
-    @property
-    def size_totals(self):
-        return cache_sync_size_totals(self.sync)

src/borg/testsuite/cache.py

@@ -10,156 +10,11 @@ from .key import TestKey
 from ..archive import Statistics
 from ..cache import AdHocCache
 from ..crypto.key import AESOCBRepoKey
-from ..hashindex import ChunkIndex, CacheSynchronizer
+from ..hashindex import ChunkIndex
 from ..manifest import Manifest
 from ..repository3 import Repository3
 
 
-class TestCacheSynchronizer:
-    @pytest.fixture
-    def index(self):
-        return ChunkIndex()
-
-    @pytest.fixture
-    def sync(self, index):
-        return CacheSynchronizer(index)
-
-    def test_no_chunks(self, index, sync):
-        data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": []})
-        sync.feed(data)
-        assert not len(index)
-
-    def test_simple(self, index, sync):
-        data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
-        sync.feed(data)
-        assert len(index) == 2
-        assert index[H(1)] == (1, 1)
-        assert index[H(2)] == (1, 2)
-
-    def test_multiple(self, index, sync):
-        data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
-        data += packb({"xattrs": {"security.foo": "bar", "chunks": "123456"}, "stuff": [(1, 2, 3)]})
-        data += packb(
-            {
-                "xattrs": {"security.foo": "bar", "chunks": "123456"},
-                "chunks": [(H(1), 1), (H(2), 2)],
-                "stuff": [(1, 2, 3)],
-            }
-        )
-        data += packb({"chunks": [(H(3), 1)]})
-        data += packb({"chunks": [(H(1), 1)]})
-
-        part1 = data[:70]
-        part2 = data[70:120]
-        part3 = data[120:]
-        sync.feed(part1)
-        sync.feed(part2)
-        sync.feed(part3)
-        assert len(index) == 3
-        assert index[H(1)] == (3, 1)
-        assert index[H(2)] == (2, 2)
-        assert index[H(3)] == (1, 1)
-
-    @pytest.mark.parametrize(
-        "elem,error",
-        (
-            ({1: 2}, "Unexpected object: map"),
-            (
-                bytes(213),
-                ["Unexpected bytes in chunks structure", "Incorrect key length"],  # structure 2/3
-            ),  # structure 3/3
-            (1, "Unexpected object: integer"),
-            (1.0, "Unexpected object: double"),
-            (True, "Unexpected object: true"),
-            (False, "Unexpected object: false"),
-            (None, "Unexpected object: nil"),
-        ),
-        ids=["map", "bytes", "int", "double", "true", "false", "none"],
-    )
-    @pytest.mark.parametrize(
-        "structure",
-        (lambda elem: {"chunks": elem}, lambda elem: {"chunks": [elem]}, lambda elem: {"chunks": [(elem, 1)]}),
-    )
-    def test_corrupted(self, sync, structure, elem, error):
-        packed = packb(structure(elem))
-        with pytest.raises(ValueError) as excinfo:
-            sync.feed(packed)
-        if isinstance(error, str):
-            error = [error]
-        possible_errors = ["cache_sync_feed failed: " + error for error in error]
-        assert str(excinfo.value) in possible_errors
-
-    @pytest.mark.parametrize(
-        "data,error",
-        (
-            # Incorrect tuple length
-            ({"chunks": [(bytes(32), 2, 3, 4)]}, "Invalid chunk list entry length"),
-            ({"chunks": [(bytes(32),)]}, "Invalid chunk list entry length"),
-            # Incorrect types
-            ({"chunks": [(1, 2)]}, "Unexpected object: integer"),
-            ({"chunks": [(1, bytes(32))]}, "Unexpected object: integer"),
-            ({"chunks": [(bytes(32), 1.0)]}, "Unexpected object: double"),
-        ),
-    )
-    def test_corrupted_ancillary(self, index, sync, data, error):
-        packed = packb(data)
-        with pytest.raises(ValueError) as excinfo:
-            sync.feed(packed)
-        assert str(excinfo.value) == "cache_sync_feed failed: " + error
-
-    def make_index_with_refcount(self, refcount):
-        index_data = io.BytesIO()
-        index_data.write(b"BORG2IDX")
-        # version
-        index_data.write((2).to_bytes(4, "little"))
-        # num_entries
-        index_data.write((1).to_bytes(4, "little"))
-        # num_buckets
-        index_data.write((1).to_bytes(4, "little"))
-        # num_empty
-        index_data.write((0).to_bytes(4, "little"))
-        # key_size
-        index_data.write((32).to_bytes(4, "little"))
-        # value_size
-        index_data.write((3 * 4).to_bytes(4, "little"))
-        # reserved
-        index_data.write(bytes(1024 - 32))
-
-        index_data.write(H(0))
-        index_data.write(refcount.to_bytes(4, "little"))
-        index_data.write((1234).to_bytes(4, "little"))
-        index_data.write((5678).to_bytes(4, "little"))
-
-        index_data.seek(0)
-        index = ChunkIndex.read(index_data)
-        return index
-
-    def test_corrupted_refcount(self):
-        index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE + 1)
-        sync = CacheSynchronizer(index)
-        data = packb({"chunks": [(H(0), 1)]})
-        with pytest.raises(ValueError) as excinfo:
-            sync.feed(data)
-        assert str(excinfo.value) == "cache_sync_feed failed: invalid reference count"
-
-    def test_refcount_max_value(self):
-        index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE)
-        sync = CacheSynchronizer(index)
-        data = packb({"chunks": [(H(0), 1)]})
-        sync.feed(data)
-        assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
-
-    def test_refcount_one_below_max_value(self):
-        index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE - 1)
-        sync = CacheSynchronizer(index)
-        data = packb({"chunks": [(H(0), 1)]})
-        sync.feed(data)
-        # Incremented to maximum
-        assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
-        sync.feed(data)
-        assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
-
-
 class TestAdHocCache:
     @pytest.fixture
     def repository(self, tmpdir):