Merge pull request #2309 from ThomasWaldmann/fix-2304

clamp (nano)second values to unproblematic range, fixes #2304

src/borg/archive.py

@@ -33,6 +33,7 @@ from .helpers import format_time, format_timedelta, format_file_size, file_statu
 from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
 from .helpers import StableDict
 from .helpers import bin_to_hex
+from .helpers import safe_ns
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
 from .helpers import PathPrefixPattern, FnmatchPattern
 from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec
@@ -786,15 +787,15 @@ Utilization of max. archive size: {csize_max:.0%}
             mode=st.st_mode,
             uid=st.st_uid,
             gid=st.st_gid,
-            mtime=st.st_mtime_ns,
+            mtime=safe_ns(st.st_mtime_ns),
         )
         # borg can work with archives only having mtime (older attic archives do not have
         # atime/ctime). it can be useful to omit atime/ctime, if they change without the
         # file content changing - e.g. to get better metadata deduplication.
         if not self.noatime:
-            attrs['atime'] = st.st_atime_ns
+            attrs['atime'] = safe_ns(st.st_atime_ns)
         if not self.noctime:
-            attrs['ctime'] = st.st_ctime_ns
+            attrs['ctime'] = safe_ns(st.st_ctime_ns)
         if self.numeric_owner:
             attrs['user'] = attrs['group'] = None
         else:

src/borg/cache.py

@@ -17,6 +17,7 @@ from .helpers import Error
 from .helpers import get_cache_dir, get_security_dir
 from .helpers import bin_to_hex
 from .helpers import format_file_size
+from .helpers import safe_ns
 from .helpers import yes
 from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
@@ -591,6 +592,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
     def memorize_file(self, path_hash, st, ids):
         if not (self.do_files and stat.S_ISREG(st.st_mode)):
             return
-        entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=st.st_mtime_ns, chunk_ids=ids)
+        mtime_ns = safe_ns(st.st_mtime_ns)
+        entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=mtime_ns, chunk_ids=ids)
         self.files[path_hash] = msgpack.packb(entry)
-        self._newest_mtime = max(self._newest_mtime or 0, st.st_mtime_ns)
+        self._newest_mtime = max(self._newest_mtime or 0, mtime_ns)

src/borg/helpers.py

@@ -665,7 +665,7 @@ def timestamp(s):
     """Convert a --timestamp=s argument to a datetime object"""
     try:
         # is it pointing to a file / directory?
-        ts = os.stat(s).st_mtime
+        ts = safe_s(os.stat(s).st_mtime)
         return datetime.utcfromtimestamp(ts)
     except OSError:
         # didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support.
@@ -818,12 +818,34 @@ def SortBySpec(text):
     return text.replace('timestamp', 'ts')
 
 
+# Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
+# As they are crap anyway, nothing is lost if we just clamp them to the max valid value.
+# msgpack can only pack uint64. datetime is limited to year 9999.
+MAX_NS = 18446744073000000000  # less than 2**64 - 1 ns. also less than y9999.
+MAX_S = MAX_NS // 1000000000
+
+
+def safe_s(ts):
+    if 0 <= ts <= MAX_S:
+        return ts
+    elif ts < 0:
+        return 0
+    else:
+        return MAX_S
+
+
+def safe_ns(ts):
+    if 0 <= ts <= MAX_NS:
+        return ts
+    elif ts < 0:
+        return 0
+    else:
+        return MAX_NS
+
+
 def safe_timestamp(item_timestamp_ns):
-    try:
-        return datetime.fromtimestamp(item_timestamp_ns / 1e9)
-    except OverflowError:
-        # likely a broken file time and datetime did not want to go beyond year 9999
-        return datetime(9999, 12, 31, 23, 59, 59)
+    t_ns = safe_ns(item_timestamp_ns)
+    return datetime.fromtimestamp(t_ns / 1e9)
 
 
 def format_time(t):

src/borg/testsuite/helpers.py

@@ -27,6 +27,7 @@ from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2
 from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
 from ..helpers import swidth_slice
 from ..helpers import chunkit
+from ..helpers import safe_ns, safe_s
 
 from . import BaseTestCase, FakeInputs
 
@@ -1221,3 +1222,18 @@ def test_swidth_slice_mixed_characters():
     string = '나윤a선나윤선나윤선나윤선나윤선'
     assert swidth_slice(string, 5) == '나윤a'
     assert swidth_slice(string, 6) == '나윤a'
+
+
+def test_safe_timestamps():
+    # ns fit into uint64
+    assert safe_ns(2 ** 64) < 2 ** 64
+    assert safe_ns(-1) == 0
+    # s are so that their ns conversion fits into uint64
+    assert safe_s(2 ** 64) * 1000000000 < 2 ** 64
+    assert safe_s(-1) == 0
+    # datetime won't fall over its y10k problem
+    beyond_y10k = 2 ** 100
+    with pytest.raises(OverflowError):
+        datetime.utcfromtimestamp(beyond_y10k)
+    assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31)
+    assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31)