Merge pull request #1377 from enkore/issue/1060

Use atomic file writing and updating for configuration and key files

src/borg/cache.py

@@ -19,6 +19,7 @@ from .helpers import yes
 from .item import Item
 from .key import PlaintextKey
 from .locking import UpgradableLock
+from .platform import SaveFile
 from .remote import cache_if_remote
 
 ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize')
@@ -141,11 +142,11 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         config.set('cache', 'version', '1')
         config.set('cache', 'repository', self.repository.id_str)
         config.set('cache', 'manifest', '')
-        with open(os.path.join(self.path, 'config'), 'w') as fd:
+        with SaveFile(os.path.join(self.path, 'config')) as fd:
             config.write(fd)
         ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
         os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
-        with open(os.path.join(self.path, 'files'), 'wb') as fd:
+        with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
             pass  # empty file
 
     def _do_open(self):
@@ -212,7 +213,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         if not self.txn_active:
             return
         if self.files is not None:
-            with open(os.path.join(self.path, 'files'), 'wb') as fd:
+            with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
                 for path_hash, item in self.files.items():
                     # Discard cached files with the newest mtime to avoid
                     # issues with filesystem snapshots and mtime precision
@@ -223,7 +224,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         self.config.set('cache', 'timestamp', self.manifest.timestamp)
         self.config.set('cache', 'key_type', str(self.key.TYPE))
         self.config.set('cache', 'previous_location', self.repository._location.canonical_path())
-        with open(os.path.join(self.path, 'config'), 'w') as fd:
+        with SaveFile(os.path.join(self.path, 'config')) as fd:
             self.config.write(fd)
         self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8'))
         os.rename(os.path.join(self.path, 'txn.active'),

src/borg/key.py

@@ -22,6 +22,7 @@ from .helpers import get_keys_dir
 from .helpers import bin_to_hex
 from .helpers import CompressionDecider2, CompressionSpec
 from .item import Key, EncryptedKey
+from .platform import SaveFile
 
 
 PREFIX = b'\0' * 8
@@ -470,7 +471,7 @@ class KeyfileKey(KeyfileKeyBase):
 
     def save(self, target, passphrase):
         key_data = self._save(passphrase)
-        with open(target, 'w') as fd:
+        with SaveFile(target) as fd:
             fd.write('%s %s\n' % (self.FILE_ID, bin_to_hex(self.repository_id)))
             fd.write(key_data)
             fd.write('\n')

src/borg/platform/__init__.py

@@ -8,7 +8,7 @@ Public APIs are documented in platform.base.
 
 from .base import acl_get, acl_set
 from .base import set_flags, get_flags
-from .base import SyncFile, sync_dir, fdatasync
+from .base import SaveFile, SyncFile, sync_dir, fdatasync
 from .base import swidth, API_VERSION
 
 if sys.platform.startswith('linux'):  # pragma: linux only

src/borg/platform/base.py

@@ -80,8 +80,9 @@ class SyncFile:
     TODO: A Windows implementation should use CreateFile with FILE_FLAG_WRITE_THROUGH.
     """
 
-    def __init__(self, path):
-        self.fd = open(path, 'xb')
+    def __init__(self, path, binary=False):
+        mode = 'xb' if binary else 'x'
+        self.fd = open(path, mode)
         self.fileno = self.fd.fileno()
 
     def __enter__(self):
@@ -112,6 +113,43 @@ class SyncFile:
         platform.sync_dir(os.path.dirname(self.fd.name))
 
 
+class SaveFile:
+    """
+    Update file contents atomically.
+
+    Must be used as a context manager (defining the scope of the transaction).
+
+    On a journaling file system the file contents are always updated
+    atomically and won't become corrupted, even on power failures or
+    crashes (for caveats see SyncFile).
+    """
+
+    SUFFIX = '.tmp'
+
+    def __init__(self, path, binary=False):
+        self.binary = binary
+        self.path = path
+        self.tmppath = self.path + self.SUFFIX
+
+    def __enter__(self):
+        from .. import platform
+        try:
+            os.unlink(self.tmppath)
+        except FileNotFoundError:
+            pass
+        self.fd = platform.SyncFile(self.tmppath, self.binary)
+        return self.fd
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        from .. import platform
+        self.fd.close()
+        if exc_type is not None:
+            os.unlink(self.tmppath)
+            return
+        os.replace(self.tmppath, self.path)
+        platform.sync_dir(os.path.dirname(self.path))
+
+
 def swidth(s):
     """terminal output width of string <s>
 

src/borg/platform/linux.pyx

@@ -228,8 +228,8 @@ class SyncFile(BaseSyncFile):
     disk in the immediate future.
     """
 
-    def __init__(self, path):
-        super().__init__(path)
+    def __init__(self, path, binary=False):
+        super().__init__(path, binary)
         self.offset = 0
         self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
         self.last_sync = 0

src/borg/repository.py

@@ -23,7 +23,7 @@ from .helpers import ProgressIndicatorPercent
 from .helpers import bin_to_hex
 from .locking import UpgradableLock, LockError, LockErrorT
 from .lrucache import LRUCache
-from .platform import SyncFile, sync_dir
+from .platform import SaveFile, SyncFile, sync_dir
 
 MAX_OBJECT_SIZE = 20 * 1024 * 1024
 MAGIC = b'BORG_SEG'
@@ -171,7 +171,7 @@ class Repository:
 
     def save_config(self, path, config):
         config_path = os.path.join(path, 'config')
-        with open(config_path, 'w') as fd:
+        with SaveFile(config_path) as fd:
             config.write(fd)
 
     def save_key(self, keydata):
@@ -784,7 +784,7 @@ class LoggedIO:
                 if not os.path.exists(dirname):
                     os.mkdir(dirname)
                     sync_dir(os.path.join(self.path, 'data'))
-            self._write_fd = SyncFile(self.segment_filename(self.segment))
+            self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True)
             self._write_fd.write(MAGIC)
             self.offset = MAGIC_LEN
         return self._write_fd