Merge pull request #1113 from enkore/feature/pltstr

platform_base should use platform-provided implementations when needed

setup.py

@@ -45,10 +45,22 @@ compress_source = 'src/borg/compress.pyx'
 crypto_source = 'src/borg/crypto.pyx'
 chunker_source = 'src/borg/chunker.pyx'
 hashindex_source = 'src/borg/hashindex.pyx'
-platform_posix_source = 'src/borg/platform_posix.pyx'
-platform_linux_source = 'src/borg/platform_linux.pyx'
-platform_darwin_source = 'src/borg/platform_darwin.pyx'
-platform_freebsd_source = 'src/borg/platform_freebsd.pyx'
+platform_posix_source = 'src/borg/platform/posix.pyx'
+platform_linux_source = 'src/borg/platform/linux.pyx'
+platform_darwin_source = 'src/borg/platform/darwin.pyx'
+platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
+
+cython_sources = [
+    compress_source,
+    crypto_source,
+    chunker_source,
+    hashindex_source,
+
+    platform_posix_source,
+    platform_linux_source,
+    platform_freebsd_source,
+    platform_darwin_source,
+]
 
 try:
     from Cython.Distutils import build_ext
@@ -56,7 +68,7 @@ try:
 
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            for src in glob('src/borg/*.pyx'):
+            for src in cython_sources:
                 cython_compiler.compile(src, cython_compiler.default_options)
             super().__init__(*args, **kwargs)
 
@@ -66,10 +78,10 @@ try:
                 'src/borg/crypto.c',
                 'src/borg/chunker.c', 'src/borg/_chunker.c',
                 'src/borg/hashindex.c', 'src/borg/_hashindex.c',
-                'src/borg/platform_posix.c',
-                'src/borg/platform_linux.c',
-                'src/borg/platform_freebsd.c',
-                'src/borg/platform_darwin.c',
+                'src/borg/platform/posix.c',
+                'src/borg/platform/linux.c',
+                'src/borg/platform/freebsd.c',
+                'src/borg/platform/darwin.c',
             ])
             super().make_distribution()
 
@@ -295,14 +307,14 @@ if not on_rtd:
     Extension('borg.hashindex', [hashindex_source])
 ]
     if sys.platform.startswith(('linux', 'freebsd', 'darwin')):
-        ext_modules.append(Extension('borg.platform_posix', [platform_posix_source]))
+        ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))
 
     if sys.platform == 'linux':
-        ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
+        ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']))
     elif sys.platform.startswith('freebsd'):
-        ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
+        ext_modules.append(Extension('borg.platform.freebsd', [platform_freebsd_source]))
     elif sys.platform == 'darwin':
-        ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
+        ext_modules.append(Extension('borg.platform.darwin', [platform_darwin_source]))
 
 setup(
     name='borgbackup',

src/borg/platform.py

@@ -1,10 +0,0 @@
-import sys
-
-from .platform_base import acl_get, acl_set, SyncFile, sync_dir, set_flags, get_flags, swidth, API_VERSION
-
-if sys.platform.startswith('linux'):  # pragma: linux only
-    from .platform_linux import acl_get, acl_set, SyncFile, set_flags, get_flags, swidth, API_VERSION
-elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
-    from .platform_freebsd import acl_get, acl_set, swidth, API_VERSION
-elif sys.platform == 'darwin':  # pragma: darwin only
-    from .platform_darwin import acl_get, acl_set, swidth, API_VERSION

src/borg/platform/__init__.py

@@ -0,0 +1,24 @@
+import sys
+
+"""
+Platform-specific APIs.
+
+Public APIs are documented in platform.base.
+"""
+
+from .base import acl_get, acl_set
+from .base import set_flags, get_flags
+from .base import SyncFile, sync_dir, fdatasync
+from .base import swidth, API_VERSION
+
+if sys.platform.startswith('linux'):  # pragma: linux only
+    from .linux import acl_get, acl_set
+    from .linux import set_flags, get_flags
+    from .linux import SyncFile
+    from .linux import swidth, API_VERSION
+elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
+    from .freebsd import acl_get, acl_set
+    from .freebsd import swidth, API_VERSION
+elif sys.platform == 'darwin':  # pragma: darwin only
+    from .darwin import acl_get, acl_set
+    from .darwin import swidth, API_VERSION

src/borg/platform_base.py → src/borg/platform/base.py

@@ -1,5 +1,17 @@
 import os
 
+"""
+platform base module
+====================
+
+Contains platform API implementations based on what Python itself provides. More specific
+APIs are stubs in this module.
+
+When functions in this module use platform APIs themselves they access the public
+platform API: that way platform APIs provided by the platform-specific support module
+are correctly composed into the base functionality.
+"""
+
 API_VERSION = 3
 
 fdatasync = getattr(os, 'fdatasync', os.fsync)
@@ -80,16 +92,18 @@ class SyncFile:
         Synchronize file contents. Everything written prior to sync() must become durable before anything written
         after sync().
         """
+        from .. import platform
         self.fd.flush()
-        fdatasync(self.fileno)
+        platform.fdatasync(self.fileno)
         if hasattr(os, 'posix_fadvise'):
             os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED)
 
     def close(self):
         """sync() and close."""
+        from .. import platform
         self.sync()
         self.fd.close()
-        sync_dir(os.path.dirname(self.fd.name))
+        platform.sync_dir(os.path.dirname(self.fd.name))
 
 
 def swidth(s):

src/borg/platform_darwin.pyx → src/borg/platform/darwin.pyx

@@ -1,8 +1,8 @@
 import os
 
-from .helpers import user2uid, group2gid
-from .helpers import safe_decode, safe_encode
-from .platform_posix import swidth
+from ..helpers import user2uid, group2gid
+from ..helpers import safe_decode, safe_encode
+from .posix import swidth
 
 API_VERSION = 3
 

src/borg/platform_freebsd.pyx → src/borg/platform/freebsd.pyx

@@ -1,8 +1,8 @@
 import os
 
-from .helpers import posix_acl_use_stored_uid_gid
-from .helpers import safe_encode, safe_decode
-from .platform_posix import swidth
+from ..helpers import posix_acl_use_stored_uid_gid
+from ..helpers import safe_encode, safe_decode
+from .posix import swidth
 
 API_VERSION = 3
 

src/borg/platform_linux.pyx → src/borg/platform/linux.pyx

@@ -3,11 +3,11 @@ import re
 import resource
 import stat
 
-from .helpers import posix_acl_use_stored_uid_gid
-from .helpers import user2uid, group2gid
-from .helpers import safe_decode, safe_encode
-from .platform_base import SyncFile as BaseSyncFile
-from .platform_posix import swidth
+from ..helpers import posix_acl_use_stored_uid_gid
+from ..helpers import user2uid, group2gid
+from ..helpers import safe_decode, safe_encode
+from .base import SyncFile as BaseSyncFile
+from .posix import swidth
 
 from libc cimport errno
 from libc.stdint cimport int64_t

src/borg/repository.py

@@ -55,7 +55,7 @@ class Repository:
     established by a COMMIT.
 
     Note that the COMMIT can't establish consistency by itself, but only manages to do so with proper support from
-    the platform (including the hardware). See platform_base.SyncFile for details.
+    the platform (including the hardware). See platform.base.SyncFile for details.
 
     A PUT inserts a key-value pair. The value is stored in the log entry, hence the repository implements
     full data logging, meaning that all data is consistent, not just metadata (which is common in file systems).

src/borg/testsuite/platform.py

@@ -102,7 +102,7 @@ class PlatformLinuxTestCase(BaseTestCase):
         self.assert_in(group_entry_numeric, acl_access_numeric)
 
     def test_utils(self):
-        from ..platform_linux import acl_use_local_uid_gid
+        from ..platform.linux import acl_use_local_uid_gid
         self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-')
         self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-')
         self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-')