include item birthtime in archive (where available) (#3313)

include item birthtime in archive, fixes #3272

* use `safe_ns` when reading birthtime into attributes
* proper order for `birthtime` in `ITEM_KEYS` list
* use `bigint` wrapper for consistency
* Add tests to verify that birthtime is normally preserved, but not preserved when `--nobirthtime` is passed to `borg create`.

src/borg/archive.py

@@ -280,7 +280,7 @@ class Archive:
         """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable."""
 
     def __init__(self, repository, key, manifest, name, cache=None, create=False,
-                 checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobsdflags=False,
+                 checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobirthtime=False, nobsdflags=False,
                  progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None,
                  consider_part_files=False, log_json=False):
         self.cwd = os.getcwd()
@@ -298,6 +298,7 @@ class Archive:
         self.numeric_owner = numeric_owner
         self.noatime = noatime
         self.noctime = noctime
+        self.nobirthtime = nobirthtime
         self.nobsdflags = nobsdflags
         assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
         if start is None:
@@ -683,6 +684,18 @@ Utilization of max. archive size: {csize_max:.0%}
         else:
             # old archives only had mtime in item metadata
             atime = mtime
+        if 'birthtime' in item:
+            birthtime = item.birthtime
+            try:
+                # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
+                # See utimes(2) on either of the BSDs for details.
+                if fd:
+                    os.utime(fd, None, ns=(atime, birthtime))
+                else:
+                    os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False)
+            except OSError:
+                # some systems don't support calling utime on a symlink
+                pass
         try:
             if fd:
                 os.utime(fd, None, ns=(atime, mtime))
@@ -822,6 +835,9 @@ Utilization of max. archive size: {csize_max:.0%}
             attrs['atime'] = safe_ns(st.st_atime_ns)
         if not self.noctime:
             attrs['ctime'] = safe_ns(st.st_ctime_ns)
+        if not self.nobirthtime and hasattr(st, 'st_birthtime'):
+            # sadly, there's no stat_result.st_birthtime_ns
+            attrs['birthtime'] = safe_ns(int(st.st_birthtime * 10**9))
         if self.numeric_owner:
             attrs['user'] = attrs['group'] = None
         else:

src/borg/archiver.py

@@ -536,7 +536,7 @@ class Archiver:
                        lock_wait=self.lock_wait, permit_adhoc_cache=args.no_cache_sync) as cache:
                 archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
                                   create=True, checkpoint_interval=args.checkpoint_interval,
-                                  numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime,
+                                  numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, nobirthtime=args.nobirthtime,
                                   nobsdflags=args.nobsdflags, progress=args.progress,
                                   chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic,
                                   log_json=args.log_json)
@@ -2964,6 +2964,8 @@ class Archiver:
                               help='do not store atime into archive')
         fs_group.add_argument('--noctime', dest='noctime', action='store_true',
                               help='do not store ctime into archive')
+        fs_group.add_argument('--nobirthtime', dest='nobirthtime', action='store_true',
+                              help='do not store birthtime (creation date) into archive')
         fs_group.add_argument('--nobsdflags', dest='nobsdflags', action='store_true',
                               help='do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive')
         fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true',

src/borg/constants.py

@@ -1,6 +1,6 @@
 # this set must be kept complete, otherwise the RobustUnpacker might malfunction:
 ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master',
-                       'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'size',
+                       'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size',
                        'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
                        'part'])
 

src/borg/item.pyx

@@ -162,6 +162,7 @@ class Item(PropDict):
     atime = PropDict._make_property('atime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
     ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
     mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
+    birthtime = PropDict._make_property('birthtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
 
     # size is only present for items with a chunk list and then it is sum(chunk_sizes)
     # compatibility note: this is a new feature, in old archives size will be missing.

src/borg/testsuite/__init__.py

@@ -116,6 +116,28 @@ def is_utime_fully_supported():
         return False
 
 
+@functools.lru_cache()
+def is_birthtime_fully_supported():
+    if not hasattr(os.stat_result, 'st_birthtime'):
+        return False
+    with unopened_tempfile() as filepath:
+        # Some filesystems (such as SSHFS) don't support utime on symlinks
+        if are_symlinks_supported():
+            os.symlink('something', filepath)
+        else:
+            open(filepath, 'w').close()
+        try:
+            birthtime, mtime, atime = 946598400, 946684800, 946771200
+            os.utime(filepath, (atime, birthtime), follow_symlinks=False)
+            os.utime(filepath, (atime, mtime), follow_symlinks=False)
+            new_stats = os.stat(filepath, follow_symlinks=False)
+            if new_stats.st_birthtime == birthtime and new_stats.st_mtime == mtime and new_stats.st_atime == atime:
+                return True
+        except OSError as err:
+            pass
+        return False
+
+
 def no_selinux(x):
     # selinux fails our FUSE tests, thus ignore selinux xattrs
     SELINUX_KEY = 'security.selinux'

src/borg/testsuite/archiver.py

@@ -53,7 +53,7 @@ from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse
 from . import BaseTestCase, changedir, environment_variable, no_selinux
-from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported
+from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported
 from .platform import fakeroot_detected
 from .upgrader import attic_repo
 from . import key
@@ -493,6 +493,39 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             # it touched the input file's atime while backing it up
             assert sto.st_atime_ns == atime * 1e9
 
+    @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime')
+    @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime')
+    def test_birthtime(self):
+        self.create_test_files()
+        birthtime, mtime, atime = 946598400, 946684800, 946771200
+        os.utime('input/file1', (atime, birthtime))
+        os.utime('input/file1', (atime, mtime))
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', self.repository_location + '::test', 'input')
+        with changedir('output'):
+            self.cmd('extract', self.repository_location + '::test')
+        sti = os.stat('input/file1')
+        sto = os.stat('output/input/file1')
+        assert int(sti.st_birthtime * 1e9) == int(sto.st_birthtime * 1e9) == birthtime * 1e9
+        assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9
+
+    @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime')
+    @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime')
+    def test_nobirthtime(self):
+        self.create_test_files()
+        birthtime, mtime, atime = 946598400, 946684800, 946771200
+        os.utime('input/file1', (atime, birthtime))
+        os.utime('input/file1', (atime, mtime))
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', '--nobirthtime', self.repository_location + '::test', 'input')
+        with changedir('output'):
+            self.cmd('extract', self.repository_location + '::test')
+        sti = os.stat('input/file1')
+        sto = os.stat('output/input/file1')
+        assert int(sti.st_birthtime * 1e9) == birthtime * 1e9
+        assert int(sto.st_birthtime * 1e9) == mtime * 1e9
+        assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9
+
     def _extract_repository_id(self, path):
         with Repository(self.repository_path) as repository:
             return repository.id