Merge pull request #7026 from ThomasWaldmann/empty-zero-unknown

metadata: differentiate between empty/zero and unknown

src/borg/archive.py

@@ -920,11 +920,7 @@ Duration: {0.duration}
                     if not symlink:
                         os.chmod(path, item.mode)
             mtime = item.mtime
-            if "atime" in item:
-                atime = item.atime
-            else:
-                # old archives only had mtime in item metadata
-                atime = mtime
+            atime = item.atime if "atime" in item else mtime
             if "birthtime" in item:
                 birthtime = item.birthtime
                 try:
@@ -947,10 +943,10 @@ Duration: {0.duration}
                 pass
             if not self.noacls:
                 acl_set(path, item, self.numeric_ids, fd=fd)
-            if not self.noxattrs:
+            if not self.noxattrs and "xattrs" in item:
                 # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include
                 # the Linux capabilities in the "security.capability" attribute.
-                warning = xattr.set_all(fd or path, item.get("xattrs", {}), follow_symlinks=False)
+                warning = xattr.set_all(fd or path, item.xattrs, follow_symlinks=False)
                 if warning:
                     set_ec(EXIT_WARNING)
             # bsdflags include the immutable flag and need to be set last:
@@ -1119,10 +1115,12 @@ class MetadataCollector:
         self.nobirthtime = nobirthtime
 
     def stat_simple_attrs(self, st):
-        attrs = dict(mode=st.st_mode, uid=st.st_uid, gid=st.st_gid, mtime=safe_ns(st.st_mtime_ns))
+        attrs = {}
+        attrs["mode"] = st.st_mode
         # borg can work with archives only having mtime (very old borg archives do not have
         # atime/ctime). it can be useful to omit atime/ctime, if they change without the
         # file content changing - e.g. to get better metadata deduplication.
+        attrs["mtime"] = safe_ns(st.st_mtime_ns)
         if not self.noatime:
             attrs["atime"] = safe_ns(st.st_atime_ns)
         if not self.noctime:
@@ -1130,6 +1128,8 @@ class MetadataCollector:
         if not self.nobirthtime and hasattr(st, "st_birthtime"):
             # sadly, there's no stat_result.st_birthtime_ns
             attrs["birthtime"] = safe_ns(int(st.st_birthtime * 10**9))
+        attrs["uid"] = st.st_uid
+        attrs["gid"] = st.st_gid
         if not self.numeric_ids:
             user = uid2user(st.st_uid)
             if user is not None:
@@ -1144,13 +1144,11 @@ class MetadataCollector:
         if not self.noflags:
             with backup_io("extended stat (flags)"):
                 flags = get_flags(path, st, fd=fd)
-            if flags:
-                attrs["bsdflags"] = flags
+            attrs["bsdflags"] = flags
         if not self.noxattrs:
             with backup_io("extended stat (xattrs)"):
                 xattrs = xattr.get_all(fd or path, follow_symlinks=False)
-            if xattrs:
-                attrs["xattrs"] = StableDict(xattrs)
+            attrs["xattrs"] = StableDict(xattrs)
         if not self.noacls:
             with backup_io("extended stat (ACLs)"):
                 acl_get(path, attrs, st, self.numeric_ids, fd=fd)

src/borg/helpers/parseformat.py

@@ -834,7 +834,7 @@ class ItemFormatter(BaseFormatter):
         item_data["source"] = source
         item_data["linktarget"] = source
         item_data["hlid"] = hlid
-        item_data["flags"] = item.get("bsdflags", 0)
+        item_data["flags"] = item.get("bsdflags")  # int if flags known, else (if flags unknown) None
         for key in self.used_call_keys:
             item_data[key] = self.call_keys[key](item)
         return item_data

src/borg/platform/darwin.pyx

@@ -81,6 +81,7 @@ def setxattr(path, name, value, *, follow_symlinks=False):
 def _remove_numeric_id_if_possible(acl):
     """Replace the user/group field with the local uid/gid if possible
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in safe_decode(acl).split('\n'):
         if entry:
@@ -98,6 +99,7 @@ def _remove_numeric_id_if_possible(acl):
 def _remove_non_numeric_identifier(acl):
     """Remove user and group names from the acl
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in safe_decode(acl).split('\n'):
         if entry:
@@ -113,22 +115,20 @@ def _remove_non_numeric_identifier(acl):
 def acl_get(path, item, st, numeric_ids=False, fd=None):
     cdef acl_t acl = NULL
     cdef char *text = NULL
-    if isinstance(path, str):
-        path = os.fsencode(path)
     try:
         if fd is not None:
             acl = acl_get_fd_np(fd, ACL_TYPE_EXTENDED)
         else:
+            if isinstance(path, str):
+                path = os.fsencode(path)
             acl = acl_get_link_np(path, ACL_TYPE_EXTENDED)
-        if acl == NULL:
-            return
-        text = acl_to_text(acl, NULL)
-        if text == NULL:
-            return
-        if numeric_ids:
-            item['acl_extended'] = _remove_non_numeric_identifier(text)
-        else:
-            item['acl_extended'] = text
+        if acl is not NULL:
+            text = acl_to_text(acl, NULL)
+            if text is not NULL:
+                if numeric_ids:
+                    item['acl_extended'] = _remove_non_numeric_identifier(text)
+                else:
+                    item['acl_extended'] = text
     finally:
         acl_free(text)
         acl_free(acl)
@@ -143,13 +143,12 @@ def acl_set(path, item, numeric_ids=False, fd=None):
                 acl = acl_from_text(acl_text)
             else:
                 acl = acl_from_text(<bytes>_remove_numeric_id_if_possible(acl_text))
-            if acl == NULL:
-                return
-            if isinstance(path, str):
-                path = os.fsencode(path)
-            if fd is not None:
-                acl_set_fd_np(fd, acl, ACL_TYPE_EXTENDED)
-            else:
-                acl_set_link_np(path, ACL_TYPE_EXTENDED, acl)
+            if acl is not NULL:
+                if fd is not None:
+                    acl_set_fd_np(fd, acl, ACL_TYPE_EXTENDED)
+                else:
+                    if isinstance(path, str):
+                        path = os.fsencode(path)
+                    acl_set_link_np(path, ACL_TYPE_EXTENDED, acl)
         finally:
             acl_free(acl)

src/borg/platform/freebsd.pyx

@@ -106,17 +106,19 @@ def setxattr(path, name, value, *, follow_symlinks=False):
 
 
 cdef _get_acl(p, type, item, attribute, flags, fd=None):
-    cdef acl_t acl
-    cdef char *text
-    if fd is not None:
-        acl = acl_get_fd_np(fd, type)
-    else:
-        acl = acl_get_link_np(p, type)
-    if acl:
-        text = acl_to_text_np(acl, NULL, flags)
-        if text:
-            item[attribute] = text
-            acl_free(text)
+    cdef acl_t acl = NULL
+    cdef char *text = NULL
+    try:
+        if fd is not None:
+            acl = acl_get_fd_np(fd, type)
+        else:
+            acl = acl_get_link_np(p, type)
+        if acl is not NULL:
+            text = acl_to_text_np(acl, NULL, flags)
+            if text is not NULL:
+                item[attribute] = text
+    finally:
+        acl_free(text)
         acl_free(acl)
 
 
@@ -139,26 +141,29 @@ def acl_get(path, item, st, numeric_ids=False, fd=None):
         _get_acl(path, ACL_TYPE_DEFAULT, item, 'acl_default', flags, fd=fd)
 
 
-cdef _set_acl(p, type, item, attribute, numeric_ids=False, fd=None):
-    cdef acl_t acl
+cdef _set_acl(path, type, item, attribute, numeric_ids=False, fd=None):
+    cdef acl_t acl = NULL
     text = item.get(attribute)
-    if text:
+    if text is not None:
         if numeric_ids and type == ACL_TYPE_NFS4:
             text = _nfs4_use_stored_uid_gid(text)
-        elif numeric_ids and type in(ACL_TYPE_ACCESS, ACL_TYPE_DEFAULT):
+        elif numeric_ids and type in (ACL_TYPE_ACCESS, ACL_TYPE_DEFAULT):
             text = posix_acl_use_stored_uid_gid(text)
-        acl = acl_from_text(<bytes>text)
-        if acl:
-            if fd is not None:
-                acl_set_fd_np(fd, acl, type)
-            else:
-                acl_set_link_np(p, type, acl)
+        try:
+            acl = acl_from_text(<bytes> text)
+            if acl is not NULL:
+                if fd is not None:
+                    acl_set_fd_np(fd, acl, type)
+                else:
+                    acl_set_link_np(path, type, acl)
+        finally:
             acl_free(acl)
 
 
 cdef _nfs4_use_stored_uid_gid(acl):
     """Replace the user/group field with the stored uid/gid
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in safe_decode(acl).split('\n'):
         if entry:

src/borg/platform/linux.pyx

@@ -179,6 +179,7 @@ def get_flags(path, st, fd=None):
 def acl_use_local_uid_gid(acl):
     """Replace the user/group field with the local uid/gid if possible
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in safe_decode(acl).split('\n'):
         if entry:
@@ -194,6 +195,7 @@ def acl_use_local_uid_gid(acl):
 cdef acl_append_numeric_ids(acl):
     """Extend the "POSIX 1003.1e draft standard 17" format with an additional uid/gid field
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
         if entry:
@@ -210,6 +212,7 @@ cdef acl_append_numeric_ids(acl):
 cdef acl_numeric_ids(acl):
     """Replace the "POSIX 1003.1e draft standard 17" user/group field with uid/gid
     """
+    assert isinstance(acl, bytes)
     entries = []
     for entry in _comment_re.sub('', safe_decode(acl)).split('\n'):
         if entry:
@@ -249,22 +252,25 @@ def acl_get(path, item, st, numeric_ids=False, fd=None):
             access_acl = acl_get_fd(fd)
         else:
             access_acl = acl_get_file(path, ACL_TYPE_ACCESS)
+        if access_acl is not NULL:
+            access_text = acl_to_text(access_acl, NULL)
+            if access_text is not NULL:
+                item['acl_access'] = converter(access_text)
+    finally:
+        acl_free(access_text)
+        acl_free(access_acl)
+
+    try:
         if stat.S_ISDIR(st.st_mode):
             # only directories can have a default ACL. there is no fd-based api to get it.
             default_acl = acl_get_file(path, ACL_TYPE_DEFAULT)
-        if access_acl:
-            access_text = acl_to_text(access_acl, NULL)
-            if access_text:
-                item['acl_access'] = converter(access_text)
-        if default_acl:
-            default_text = acl_to_text(default_acl, NULL)
-            if default_text:
-                item['acl_default'] = converter(default_text)
+            if default_acl is not NULL:
+                default_text = acl_to_text(default_acl, NULL)
+                if default_text is not NULL:
+                    item['acl_default'] = converter(default_text)
     finally:
         acl_free(default_text)
         acl_free(default_acl)
-        acl_free(access_text)
-        acl_free(access_acl)
 
 
 def acl_set(path, item, numeric_ids=False, fd=None):
@@ -282,10 +288,10 @@ def acl_set(path, item, numeric_ids=False, fd=None):
     else:
         converter = acl_use_local_uid_gid
     access_text = item.get('acl_access')
-    if access_text:
+    if access_text is not None:
         try:
-            access_acl = acl_from_text(<bytes>converter(access_text))
-            if access_acl:
+            access_acl = acl_from_text(<bytes> converter(access_text))
+            if access_acl is not NULL:
                 if fd is not None:
                     acl_set_fd(fd, access_acl)
                 else:
@@ -293,10 +299,10 @@ def acl_set(path, item, numeric_ids=False, fd=None):
         finally:
             acl_free(access_acl)
     default_text = item.get('acl_default')
-    if default_text:
+    if default_text is not None:
         try:
-            default_acl = acl_from_text(<bytes>converter(default_text))
-            if default_acl:
+            default_acl = acl_from_text(<bytes> converter(default_text))
+            if default_acl is not NULL:
                 # only directories can get a default ACL. there is no fd-based api to set it.
                 acl_set_file(path, ACL_TYPE_DEFAULT, default_acl)
         finally:

src/borg/platform/posix.pyx

@@ -87,8 +87,10 @@ def uid2user(uid, default=None):
 
 @lru_cache(maxsize=None)
 def user2uid(user, default=None):
+    if not user:
+        return default
     try:
-        return user and pwd.getpwnam(user).pw_uid
+        return pwd.getpwnam(user).pw_uid
     except KeyError:
         return default
 
@@ -103,8 +105,10 @@ def gid2group(gid, default=None):
 
 @lru_cache(maxsize=None)
 def group2gid(group, default=None):
+    if not group:
+        return default
     try:
-        return group and grp.getgrnam(group).gr_gid
+        return grp.getgrnam(group).gr_gid
     except KeyError:
         return default
 
@@ -112,6 +116,7 @@ def group2gid(group, default=None):
 def posix_acl_use_stored_uid_gid(acl):
     """Replace the user/group field with the stored uid/gid
     """
+    assert isinstance(acl, bytes)
     from ..helpers import safe_decode, safe_encode
     entries = []
     for entry in safe_decode(acl).split('\n'):

src/borg/upgrade.py

@@ -93,6 +93,11 @@ class UpgraderFrom12To20:
         # - 'acl' remnants of bug in attic <= 0.13
         # - 'hardlink_master' (superseded by hlid)
         new_item_dict = {key: value for key, value in item.as_dict().items() if key in ITEM_KEY_WHITELIST}
+        # remove some pointless entries older borg put in there:
+        for key in "user", "group":
+            if key in new_item_dict and new_item_dict[key] is None:
+                del new_item_dict[key]
+        assert not any(value is None for value in new_item_dict.values()), f"found None value in {new_item_dict}"
         new_item = Item(internal_dict=new_item_dict)
         new_item.get_size(memorize=True)  # if not already present: compute+remember size for items with chunks
         assert all(key in new_item for key in REQUIRED_ITEM_KEYS)