Merge pull request #4164 from motwok/platform-refactor

remove posix issues and add fixups for unsupported methods. (ex. #4154)

src/borg/archive.py

@@ -13,6 +13,7 @@ from io import BytesIO
 from itertools import groupby, zip_longest
 from shutil import get_terminal_size
 
+from .platformflags import is_win32, is_linux, is_freebsd, is_darwin
 from .logger import create_logger
 
 logger = create_logger()
@@ -29,7 +30,7 @@ from .helpers import Manifest
 from .helpers import hardlinkable
 from .helpers import ChunkIteratorFileWrapper, open_item
 from .helpers import Error, IntegrityError, set_ec
-from .helpers import uid2user, user2uid, gid2group, group2gid
+from .platform import uid2user, user2uid, gid2group, group2gid
 from .helpers import parse_timestamp, to_localtime
 from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
 from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
@@ -679,57 +680,58 @@ Utilization of max. archive size: {csize_max:.0%}
         uid = item.uid if uid is None else uid
         gid = item.gid if gid is None else gid
         # This code is a bit of a mess due to os specific differences
-        try:
-            if fd:
-                os.fchown(fd, uid, gid)
-            else:
-                os.chown(path, uid, gid, follow_symlinks=False)
-        except OSError:
-            pass
-        if fd:
-            os.fchmod(fd, item.mode)
-        elif not symlink:
-            os.chmod(path, item.mode)
-        elif has_lchmod:  # Not available on Linux
-            os.lchmod(path, item.mode)
-        mtime = item.mtime
-        if 'atime' in item:
-            atime = item.atime
-        else:
-            # old archives only had mtime in item metadata
-            atime = mtime
-        if 'birthtime' in item:
-            birthtime = item.birthtime
+        if not is_win32:
             try:
-                # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
-                # See utimes(2) on either of the BSDs for details.
                 if fd:
-                    os.utime(fd, None, ns=(atime, birthtime))
+                    os.fchown(fd, uid, gid)
                 else:
-                    os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False)
+                    os.chown(path, uid, gid, follow_symlinks=False)
             except OSError:
-                # some systems don't support calling utime on a symlink
                 pass
-        try:
             if fd:
-                os.utime(fd, None, ns=(atime, mtime))
+                os.fchmod(fd, item.mode)
+            elif not symlink:
+                os.chmod(path, item.mode)
+            elif has_lchmod:  # Not available on Linux
+                os.lchmod(path, item.mode)
+            mtime = item.mtime
+            if 'atime' in item:
+                atime = item.atime
             else:
-                os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
-        except OSError:
-            # some systems don't support calling utime on a symlink
-            pass
-        acl_set(path, item, self.numeric_owner, fd=fd)
-        # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include
-        # the Linux capabilities in the "security.capability" attribute.
-        warning = xattr.set_all(fd or path, item.get('xattrs', {}), follow_symlinks=False)
-        if warning:
-            set_ec(EXIT_WARNING)
-        # bsdflags include the immutable flag and need to be set last:
-        if not self.nobsdflags and 'bsdflags' in item:
+                # old archives only had mtime in item metadata
+                atime = mtime
+            if 'birthtime' in item:
+                birthtime = item.birthtime
+                try:
+                    # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms.
+                    # See utimes(2) on either of the BSDs for details.
+                    if fd:
+                        os.utime(fd, None, ns=(atime, birthtime))
+                    else:
+                        os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False)
+                except OSError:
+                    # some systems don't support calling utime on a symlink
+                    pass
             try:
-                set_flags(path, item.bsdflags, fd=fd)
+                if fd:
+                    os.utime(fd, None, ns=(atime, mtime))
+                else:
+                    os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
             except OSError:
+                # some systems don't support calling utime on a symlink
                 pass
+            acl_set(path, item, self.numeric_owner, fd=fd)
+            # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include
+            # the Linux capabilities in the "security.capability" attribute.
+            warning = xattr.set_all(fd or path, item.get('xattrs', {}), follow_symlinks=False)
+            if warning:
+                set_ec(EXIT_WARNING)
+            # bsdflags include the immutable flag and need to be set last:
+            if not self.nobsdflags and 'bsdflags' in item:
+                try:
+                    set_flags(path, item.bsdflags, fd=fd)
+                except OSError:
+                    pass
 
     def set_meta(self, key, value):
         metadata = self._load_meta(self.id)

src/borg/helpers/__init__.py

@@ -16,7 +16,6 @@ from .parseformat import *  # NOQA
 from .process import *  # NOQA
 from .progress import *  # NOQA
 from .time import *  # NOQA
-from .usergroup import *  # NOQA
 from .yes import *  # NOQA
 
 from .msgpack import is_slow_msgpack, int_to_bigint, bigint_to_int, get_limited_unpacker

src/borg/helpers/checks.py

@@ -1,6 +1,8 @@
 import os
+import sys
 
 from .errors import Error
+from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
 
 
 class PythonLibcTooOld(Error):
@@ -8,7 +10,10 @@ class PythonLibcTooOld(Error):
 
 
 def check_python():
-    required_funcs = {os.stat, os.utime, os.chown}
+    if is_win32:
+        required_funcs = {os.stat}
+    else:
+        required_funcs = {os.stat, os.utime, os.chown}
     if not os.supports_follow_symlinks.issuperset(required_funcs):
         raise PythonLibcTooOld
 

src/borg/helpers/parseformat.py

@@ -19,7 +19,6 @@ logger = create_logger()
 from .errors import Error
 from .fs import get_keys_dir
 from .time import OutputTimestamp, format_time, to_localtime, safe_timestamp, safe_s
-from .usergroup import uid2user
 from .. import __version__ as borg_version
 from .. import __version_tuple__ as borg_version_tuple
 from ..constants import *  # NOQA
@@ -179,7 +178,7 @@ def format_line(format, data):
 
 def replace_placeholders(text):
     """Replace placeholders in text with their values."""
-    from ..platform import fqdn, hostname
+    from ..platform import fqdn, hostname, getosusername
     current_time = datetime.now(timezone.utc)
     data = {
         'pid': os.getpid(),
@@ -188,7 +187,7 @@ def replace_placeholders(text):
         'hostname': hostname,
         'now': DatetimeWrapper(current_time.astimezone(None)),
         'utcnow': DatetimeWrapper(current_time),
-        'user': uid2user(os.getuid(), os.getuid()),
+        'user': getosusername(),
         'uuid4': str(uuid.uuid4()),
         'borgversion': borg_version,
         'borgmajor': '%d' % borg_version_tuple[:1],

src/borg/helpers/process.py

@@ -9,6 +9,7 @@ import sys
 
 from .. import __version__
 
+from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
 from ..logger import create_logger
 logger = create_logger()
 
@@ -117,7 +118,7 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
 
 
 def is_terminal(fd=sys.stdout):
-    return hasattr(fd, 'isatty') and fd.isatty() and (sys.platform != 'win32' or 'ANSICON' in os.environ)
+    return hasattr(fd, 'isatty') and fd.isatty() and (not is_win32 or 'ANSICON' in os.environ)
 
 
 def prepare_subprocess_env(system, env=None):

src/borg/helpers/usergroup.py

@@ -1,50 +0,0 @@
-import grp
-import pwd
-from functools import lru_cache
-
-
-@lru_cache(maxsize=None)
-def uid2user(uid, default=None):
-    try:
-        return pwd.getpwuid(uid).pw_name
-    except KeyError:
-        return default
-
-
-@lru_cache(maxsize=None)
-def user2uid(user, default=None):
-    try:
-        return user and pwd.getpwnam(user).pw_uid
-    except KeyError:
-        return default
-
-
-@lru_cache(maxsize=None)
-def gid2group(gid, default=None):
-    try:
-        return grp.getgrgid(gid).gr_name
-    except KeyError:
-        return default
-
-
-@lru_cache(maxsize=None)
-def group2gid(group, default=None):
-    try:
-        return group and grp.getgrnam(group).gr_gid
-    except KeyError:
-        return default
-
-
-def posix_acl_use_stored_uid_gid(acl):
-    """Replace the user/group field with the stored uid/gid
-    """
-    from .parseformat import safe_decode, safe_encode
-    entries = []
-    for entry in safe_decode(acl).split('\n'):
-        if entry:
-            fields = entry.split(':')
-            if len(fields) == 4:
-                entries.append(':'.join([fields[0], fields[3], fields[2]]))
-            else:
-                entries.append(entry)
-    return safe_encode('\n'.join(entries))

src/borg/platform/__init__.py

@@ -1,11 +1,11 @@
-import sys
-
 """
 Platform-specific APIs.
 
 Public APIs are documented in platform.base.
 """
 
+from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
+
 from .base import listxattr, getxattr, setxattr, ENOATTR
 from .base import acl_get, acl_set
 from .base import set_flags, get_flags
@@ -15,22 +15,24 @@ from .base import process_alive, get_process_id, local_pid_alive, fqdn, hostname
 
 OS_API_VERSION = API_VERSION
 
-if not sys.platform.startswith(('win32', )):
+if not is_win32:
     from .posix import process_alive, local_pid_alive
     # posix swidth implementation works for: linux, freebsd, darwin, openindiana, cygwin
     from .posix import swidth
+    from .posix import get_errno
+    from .posix import uid2user, user2uid, gid2group, group2gid, getosusername
 
-if sys.platform.startswith('linux'):  # pragma: linux only
+if is_linux:  # pragma: linux only
     from .linux import API_VERSION as OS_API_VERSION
     from .linux import listxattr, getxattr, setxattr
     from .linux import acl_get, acl_set
     from .linux import set_flags, get_flags
     from .linux import SyncFile
-elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
+elif is_freebsd:  # pragma: freebsd only
     from .freebsd import API_VERSION as OS_API_VERSION
     from .freebsd import listxattr, getxattr, setxattr
     from .freebsd import acl_get, acl_set
-elif sys.platform == 'darwin':  # pragma: darwin only
+elif is_darwin:  # pragma: darwin only
     from .darwin import API_VERSION as OS_API_VERSION
     from .darwin import listxattr, getxattr, setxattr
     from .darwin import acl_get, acl_set

src/borg/platform/darwin.pyx

@@ -2,7 +2,7 @@ import os
 
 from libc.stdint cimport uint32_t
 
-from ..helpers import user2uid, group2gid
+from .posix import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
 from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_string0
 

src/borg/platform/freebsd.pyx

@@ -1,6 +1,6 @@
 import os
 
-from ..helpers import posix_acl_use_stored_uid_gid
+from .posix import posix_acl_use_stored_uid_gid
 from ..helpers import safe_encode, safe_decode
 from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_lstring
 

src/borg/platform/linux.pyx

@@ -3,8 +3,8 @@ import re
 import stat
 import subprocess
 
-from ..helpers import posix_acl_use_stored_uid_gid
-from ..helpers import user2uid, group2gid
+from .posix import posix_acl_use_stored_uid_gid
+from .posix import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
 from .base import SyncFile as BaseSyncFile
 from .base import safe_fadvise

src/borg/platform/posix.pyx

@@ -1,5 +1,8 @@
 import errno
 import os
+import grp
+import pwd
+from functools import lru_cache
 
 from libc.errno cimport errno as c_errno
 
@@ -61,3 +64,56 @@ def local_pid_alive(pid):
             return False
         # Any other error (eg. permissions) means that the process ID refers to a live process.
         return True
+
+
+@lru_cache(maxsize=None)
+def uid2user(uid, default=None):
+    try:
+        return pwd.getpwuid(uid).pw_name
+    except KeyError:
+        return default
+
+
+@lru_cache(maxsize=None)
+def user2uid(user, default=None):
+    try:
+        return user and pwd.getpwnam(user).pw_uid
+    except KeyError:
+        return default
+
+
+@lru_cache(maxsize=None)
+def gid2group(gid, default=None):
+    try:
+        return grp.getgrgid(gid).gr_name
+    except KeyError:
+        return default
+
+
+@lru_cache(maxsize=None)
+def group2gid(group, default=None):
+    try:
+        return group and grp.getgrnam(group).gr_gid
+    except KeyError:
+        return default
+
+
+def posix_acl_use_stored_uid_gid(acl):
+    """Replace the user/group field with the stored uid/gid
+    """
+    from ..helpers import safe_decode, safe_encode
+    entries = []
+    for entry in safe_decode(acl).split('\n'):
+        if entry:
+            fields = entry.split(':')
+            if len(fields) == 4:
+                entries.append(':'.join([fields[0], fields[3], fields[2]]))
+            else:
+                entries.append(entry)
+    return safe_encode('\n'.join(entries))
+
+
+def getosusername():
+    """Return the os user name."""
+    uid = os.getuid()
+    return uid2user(uid, uid)

src/borg/platform/xattr.py

@@ -1,8 +1,6 @@
 import errno
 import os
 
-from .posix import get_errno
-
 from ..helpers import Buffer
 
 
@@ -39,6 +37,7 @@ class BufferTooSmallError(Exception):
 
 
 def _check(rv, path=None, detect_buffer_too_small=False):
+    from . import get_errno
     if rv < 0:
         e = get_errno()
         if detect_buffer_too_small and e == errno.ERANGE:

src/borg/platformflags.py

@@ -0,0 +1,12 @@
+"""
+Flags for Platform-specific APIs.
+
+Use these Flags instead of sys.platform.startswith('<OS>') or try/except.
+"""
+
+import sys
+
+is_win32 = sys.platform.startswith('win32')
+is_linux = sys.platform.startswith('linux')
+is_freebsd = sys.platform.startswith('freebsd')
+is_darwin = sys.platform.startswith('darwin')

src/borg/repository.py

@@ -652,6 +652,10 @@ class Repository:
         except OSError as os_error:
             logger.warning('Failed to check free space before committing: ' + str(os_error))
             return
+        except AttributeError:
+            # TODO move the call to statvfs to platform
+            logger.warning('Failed to check free space before committing: no statvfs method available')
+            return
         # f_bavail: even as root - don't touch the Federal Block Reserve!
         free_space = st_vfs.f_bavail * st_vfs.f_bsize
         logger.debug('check_free_space: required bytes {}, free bytes {}'.format(required_free_space, free_space))

src/borg/testsuite/__init__.py

@@ -2,7 +2,11 @@ from contextlib import contextmanager
 import filecmp
 import functools
 import os
-import posix
+try:
+    import posix
+except ImportError:
+    posix = None
+
 import stat
 import sys
 import sysconfig
@@ -44,7 +48,7 @@ except ImportError:
     has_llfuse = False
 
 # The mtime get/set precision varies on different OS and Python versions
-if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
+if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
     st_mtime_ns_round = 0
 elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
     st_mtime_ns_round = -6
@@ -95,7 +99,10 @@ def are_fifos_supported():
             os.mkfifo(filepath)
             return True
         except OSError:
-            return False
+            pass
+        except NotImplementedError:
+            pass
+        return False
 
 
 @functools.lru_cache()
@@ -113,6 +120,8 @@ def is_utime_fully_supported():
                 return True
         except OSError:
             pass
+        except NotImplementedError:
+            pass
         return False
 
 
@@ -135,6 +144,8 @@ def is_birthtime_fully_supported():
                 return True
         except OSError:
             pass
+        except NotImplementedError:
+            pass
         return False
 
 

src/borg/testsuite/hashindex.py

@@ -8,7 +8,7 @@ import zlib
 from ..hashindex import NSIndex, ChunkIndex, ChunkIndexEntry
 from .. import hashindex
 from ..crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError
-from . import BaseTestCase
+from . import BaseTestCase, unopened_tempfile
 
 # Note: these tests are part of the self test, do not use or import py.test functionality here.
 #       See borg.selftest for details. If you add/remove test methods, update SELFTEST_COUNT
@@ -54,22 +54,22 @@ class HashIndexTestCase(BaseTestCase):
         for x in range(50):
             self.assert_raises(KeyError, idx.__delitem__, H(x))
         self.assert_equal(len(idx), 50)
-        idx_name = tempfile.NamedTemporaryFile()
-        idx.write(idx_name.name)
-        del idx
-        # Verify file contents
-        with open(idx_name.name, 'rb') as fd:
-            self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha)
-        # Make sure we can open the file
-        idx = cls.read(idx_name.name)
-        self.assert_equal(len(idx), 50)
-        for x in range(50, 100):
-            self.assert_equal(idx[H(x)], make_value(x * 2))
-        idx.clear()
-        self.assert_equal(len(idx), 0)
-        idx.write(idx_name.name)
-        del idx
-        self.assert_equal(len(cls.read(idx_name.name)), 0)
+        with unopened_tempfile() as filepath:
+            idx.write(filepath)
+            del idx
+            # Verify file contents
+            with open(filepath, 'rb') as fd:
+                self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha)
+            # Make sure we can open the file
+            idx = cls.read(filepath)
+            self.assert_equal(len(idx), 50)
+            for x in range(50, 100):
+                self.assert_equal(idx[H(x)], make_value(x * 2))
+            idx.clear()
+            self.assert_equal(len(idx), 0)
+            idx.write(filepath)
+            del idx
+            self.assert_equal(len(cls.read(filepath)), 0)
 
     def test_nsindex(self):
         self._generic_test(NSIndex, lambda x: (x, x),
@@ -81,20 +81,20 @@ class HashIndexTestCase(BaseTestCase):
 
     def test_resize(self):
         n = 2000  # Must be >= MIN_BUCKETS
-        idx_name = tempfile.NamedTemporaryFile()
-        idx = NSIndex()
-        idx.write(idx_name.name)
-        initial_size = os.path.getsize(idx_name.name)
-        self.assert_equal(len(idx), 0)
-        for x in range(n):
-            idx[H(x)] = x, x
-        idx.write(idx_name.name)
-        self.assert_true(initial_size < os.path.getsize(idx_name.name))
-        for x in range(n):
-            del idx[H(x)]
-        self.assert_equal(len(idx), 0)
-        idx.write(idx_name.name)
-        self.assert_equal(initial_size, os.path.getsize(idx_name.name))
+        with unopened_tempfile() as filepath:
+            idx = NSIndex()
+            idx.write(filepath)
+            initial_size = os.path.getsize(filepath)
+            self.assert_equal(len(idx), 0)
+            for x in range(n):
+                idx[H(x)] = x, x
+            idx.write(filepath)
+            self.assert_true(initial_size < os.path.getsize(filepath))
+            for x in range(n):
+                del idx[H(x)]
+            self.assert_equal(len(idx), 0)
+            idx.write(filepath)
+            self.assert_equal(initial_size, os.path.getsize(filepath))
 
     def test_iteritems(self):
         idx = NSIndex()

src/borg/testsuite/platform.py

@@ -4,9 +4,9 @@ import random
 import shutil
 import sys
 import tempfile
-import pwd
 import unittest
 
+from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
 from ..platform import acl_get, acl_set, swidth
 from ..platform import get_process_id, process_alive
 from . import BaseTestCase, unopened_tempfile
@@ -43,11 +43,14 @@ def fakeroot_detected():
 
 
 def user_exists(username):
-    try:
-        pwd.getpwnam(username)
-        return True
-    except (KeyError, ValueError):
-        return False
+    if not is_win32:
+        import pwd
+        try:
+            pwd.getpwnam(username)
+            return True
+        except (KeyError, ValueError):
+            pass
+    return False
 
 
 @functools.lru_cache()