Merge pull request #2321 from ThomasWaldmann/revert-bigint-removal

Revert bigint removal

src/borg/cache.py

@@ -15,7 +15,7 @@ from .hashindex import ChunkIndex, ChunkIndexEntry
 from .helpers import Location
 from .helpers import Error
 from .helpers import get_cache_dir, get_security_dir
-from .helpers import bin_to_hex
+from .helpers import int_to_bigint, bigint_to_int, bin_to_hex
 from .helpers import format_file_size
 from .helpers import safe_ns
 from .helpers import yes
@@ -354,7 +354,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                     # this is to avoid issues with filesystem snapshots and mtime granularity.
                     # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
                     entry = FileCacheEntry(*msgpack.unpackb(item))
-                    if entry.age == 0 and entry.mtime < self._newest_mtime or \
+                    if entry.age == 0 and bigint_to_int(entry.mtime) < self._newest_mtime or \
                        entry.age > 0 and entry.age < ttl:
                         msgpack.pack((path_hash, entry), fd)
         pi.output('Saving cache config')
@@ -574,7 +574,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not entry:
             return None
         entry = FileCacheEntry(*msgpack.unpackb(entry))
-        if (entry.size == st.st_size and entry.mtime == st.st_mtime_ns and
+        if (entry.size == st.st_size and bigint_to_int(entry.mtime) == st.st_mtime_ns and
                 (ignore_inode or entry.inode == st.st_ino)):
             # we ignored the inode number in the comparison above or it is still same.
             # if it is still the same, replacing it in the tuple doesn't change it.
@@ -593,6 +593,6 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not (self.do_files and stat.S_ISREG(st.st_mode)):
             return
         mtime_ns = safe_ns(st.st_mtime_ns)
-        entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=mtime_ns, chunk_ids=ids)
+        entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=int_to_bigint(mtime_ns), chunk_ids=ids)
         self.files[path_hash] = msgpack.packb(entry)
         self._newest_mtime = max(self._newest_mtime or 0, mtime_ns)

src/borg/helpers.py

@@ -1332,6 +1332,24 @@ class StableDict(dict):
         return sorted(super().items())
 
 
+def bigint_to_int(mtime):
+    """Convert bytearray to int
+    """
+    if isinstance(mtime, bytes):
+        return int.from_bytes(mtime, 'little', signed=True)
+    return mtime
+
+
+def int_to_bigint(value):
+    """Convert integers larger than 64 bits to bytearray
+
+    Smaller integers are left alone
+    """
+    if value.bit_length() > 63:
+        return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True)
+    return value
+
+
 def is_slow_msgpack():
     return msgpack.Packer is msgpack.fallback.Packer
 

src/borg/item.pyx

@@ -2,6 +2,7 @@ from collections import namedtuple
 
 from .constants import ITEM_KEYS
 from .helpers import safe_encode, safe_decode
+from .helpers import bigint_to_int, int_to_bigint
 from .helpers import StableDict
 
 API_VERSION = '1.1_02'
@@ -156,9 +157,10 @@ class Item(PropDict):
     rdev = PropDict._make_property('rdev', int)
     bsdflags = PropDict._make_property('bsdflags', int)
 
-    atime = PropDict._make_property('atime', int)
-    ctime = PropDict._make_property('ctime', int)
-    mtime = PropDict._make_property('mtime', int)
+    # note: we need to keep the bigint conversion for compatibility with borg 1.0 archives.
+    atime = PropDict._make_property('atime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
+    ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
+    mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
 
     # size is only present for items with a chunk list and then it is sum(chunk_sizes)
     # compatibility note: this is a new feature, in old archives size will be missing.

src/borg/testsuite/helpers.py

@@ -20,7 +20,7 @@ from ..helpers import prune_within, prune_split
 from ..helpers import get_cache_dir, get_keys_dir, get_security_dir
 from ..helpers import is_slow_msgpack
 from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
-from ..helpers import StableDict, bin_to_hex
+from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
 from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk
 from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless
 from ..helpers import load_exclude_file, load_pattern_file
@@ -33,6 +33,18 @@ from ..helpers import safe_ns, safe_s
 from . import BaseTestCase, FakeInputs
 
 
+class BigIntTestCase(BaseTestCase):
+
+    def test_bigint(self):
+        self.assert_equal(int_to_bigint(0), 0)
+        self.assert_equal(int_to_bigint(2**63-1), 2**63-1)
+        self.assert_equal(int_to_bigint(-2**63+1), -2**63+1)
+        self.assert_equal(int_to_bigint(2**63), b'\x00\x00\x00\x00\x00\x00\x00\x80\x00')
+        self.assert_equal(int_to_bigint(-2**63), b'\x00\x00\x00\x00\x00\x00\x00\x80\xff')
+        self.assert_equal(bigint_to_int(int_to_bigint(-2**70)), -2**70)
+        self.assert_equal(bigint_to_int(int_to_bigint(2**70)), 2**70)
+
+
 def test_bin_to_hex():
     assert bin_to_hex(b'') == ''
     assert bin_to_hex(b'\x00\x01\xff') == '0001ff'

src/borg/testsuite/item.py

@@ -77,6 +77,17 @@ def test_item_int_property():
         item.mode = "invalid"
 
 
+def test_item_bigint_property():
+    item = Item()
+    small, big = 42, 2 ** 65
+    item.atime = small
+    assert item.atime == small
+    assert item.as_dict() == {'atime': small}
+    item.atime = big
+    assert item.atime == big
+    assert item.as_dict() == {'atime': b'\0' * 8 + b'\x02'}
+
+
 def test_item_user_group_none():
     item = Item()
     item.user = None