Merge branch '1.0-maint'

Vagrantfile

@@ -202,8 +202,8 @@ def install_borg(boxname)
     rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
     rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
     pip install -r requirements.d/development.txt
-    pip install -r requirements.d/fuse.txt
-    pip install -e .
+    # by using [fuse], setup.py can handle different fuse requirements:
+    pip install -e .[fuse]
   EOF
 end
 

borg/archiver.py

@@ -2120,6 +2120,14 @@ def sig_info_handler(signum, stack):  # pragma: no cover
             break
 
 
+class SIGTERMReceived(BaseException):
+    pass
+
+
+def sig_term_handler(signum, stack):
+    raise SIGTERMReceived
+
+
 def setup_signal_handlers():  # pragma: no cover
     sigs = []
     if hasattr(signal, 'SIGUSR1'):
@@ -2128,6 +2136,7 @@ def setup_signal_handlers():  # pragma: no cover
         sigs.append(signal.SIGINFO)  # kill -INFO pid (or ctrl-t)
     for sig in sigs:
         signal.signal(sig, sig_info_handler)
+    signal.signal(signal.SIGTERM, sig_term_handler)
 
 
 def main():  # pragma: no cover
@@ -2159,6 +2168,9 @@ def main():  # pragma: no cover
     except KeyboardInterrupt:
         msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo())
         exit_code = EXIT_ERROR
+    except SIGTERMReceived:
+        msg = 'Received SIGTERM.'
+        exit_code = EXIT_ERROR
     if msg:
         logger.error(msg)
     if args.show_rc:

borg/key.py

@@ -1,4 +1,4 @@
-from binascii import a2b_base64, b2a_base64
+from binascii import a2b_base64, b2a_base64, hexlify
 import configparser
 import getpass
 import os
@@ -413,16 +413,19 @@ class KeyfileKey(KeyfileKeyBase):
     FILE_ID = 'BORG_KEY'
 
     def sanity_check(self, filename, id):
-        with open(filename, 'r') as fd:
-            line = fd.readline().strip()
-            if not line.startswith(self.FILE_ID):
+        file_id = self.FILE_ID.encode() + b' '
+        repo_id = hexlify(id)
+        with open(filename, 'rb') as fd:
+            # we do the magic / id check in binary mode to avoid stumbling over
+            # decoding errors if somebody has binary files in the keys dir for some reason.
+            if fd.read(len(file_id)) != file_id:
                 raise KeyfileInvalidError(self.repository._location.canonical_path(), filename)
-            if line[len(self.FILE_ID) + 1:] != id:
+            if fd.read(len(repo_id)) != repo_id:
                 raise KeyfileMismatchError(self.repository._location.canonical_path(), filename)
             return filename
 
     def find_key(self):
-        id = self.repository.id_str
+        id = self.repository.id
         keyfile = os.environ.get('BORG_KEY_FILE')
         if keyfile:
             return self.sanity_check(keyfile, id)

borg/remote.py

@@ -424,6 +424,9 @@ class RepositoryCache(RepositoryNoCache):
 
     Caches Repository GET operations using a local temporary Repository.
     """
+    # maximum object size that will be cached, 64 kiB.
+    THRESHOLD = 2**16
+
     def __init__(self, repository):
         super().__init__(repository)
         tmppath = tempfile.mkdtemp(prefix='borg-tmp')
@@ -444,7 +447,8 @@ class RepositoryCache(RepositoryNoCache):
             except Repository.ObjectNotFound:
                 for key_, data in repository_iterator:
                     if key_ == key:
-                        self.caching_repo.put(key, data)
+                        if len(data) <= self.THRESHOLD:
+                            self.caching_repo.put(key, data)
                         yield data
                         break
         # Consume any pending requests

borg/repository.py

@@ -544,7 +544,7 @@ class LoggedIO:
         """Return the last committed segment.
         """
         for segment, filename in self.segment_iterator(reverse=True):
-            if self.is_committed_segment(filename):
+            if self.is_committed_segment(segment):
                 return segment
         return None
 
@@ -558,10 +558,14 @@ class LoggedIO:
             else:
                 break
 
-    def is_committed_segment(self, filename):
+    def is_committed_segment(self, segment):
         """Check if segment ends with a COMMIT_TAG tag
         """
-        with open(filename, 'rb') as fd:
+        try:
+            iterator = self.iter_objects(segment)
+        except IntegrityError:
+            return False
+        with open(self.segment_filename(segment), 'rb') as fd:
             try:
                 fd.seek(-self.header_fmt.size, os.SEEK_END)
             except OSError as e:
@@ -569,7 +573,22 @@ class LoggedIO:
                 if e.errno == errno.EINVAL:
                     return False
                 raise e
-            return fd.read(self.header_fmt.size) == self.COMMIT
+            if fd.read(self.header_fmt.size) != self.COMMIT:
+                return False
+        seen_commit = False
+        while True:
+            try:
+                tag, key, offset = next(iterator)
+            except IntegrityError:
+                return False
+            except StopIteration:
+                break
+            if tag == TAG_COMMIT:
+                seen_commit = True
+                continue
+            if seen_commit:
+                return False
+        return seen_commit
 
     def segment_filename(self, segment):
         return os.path.join(self.path, 'data', str(segment // self.segments_per_dir), str(segment))

borg/testsuite/repository.py

@@ -10,7 +10,7 @@ from ..hashindex import NSIndex
 from ..helpers import Location, IntegrityError
 from ..locking import UpgradableLock, LockFailed
 from ..remote import RemoteRepository, InvalidRPCMethod, ConnectionClosedWithHint
-from ..repository import Repository
+from ..repository import Repository, LoggedIO
 from . import BaseTestCase
 
 
@@ -194,6 +194,13 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
             self.assert_equal(self.repository.check(), True)
             self.assert_equal(len(self.repository), 3)
 
+    def test_ignores_commit_tag_in_data(self):
+        self.repository.put(b'0' * 32, LoggedIO.COMMIT)
+        self.reopen()
+        with self.repository:
+            io = self.repository.io
+            assert not io.is_committed_segment(io.get_latest_segment())
+
 
 class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
     def test_destroy_append_only(self):
@@ -270,7 +277,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         return set(int(key) for key in self.repository.list())
 
     def test_repair_corrupted_segment(self):
-        self.add_objects([[1, 2, 3], [4, 5, 6]])
+        self.add_objects([[1, 2, 3], [4, 5], [6]])
         self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
         self.check(status=True)
         self.corrupt_object(5)

docs/api.rst

@@ -2,55 +2,55 @@
 API Documentation
 =================
 
-.. automodule:: borg.platform
+.. automodule:: borg.archiver
     :members:
     :undoc-members:
 
-.. automodule:: borg.shellpattern
+.. automodule:: borg.upgrader
     :members:
     :undoc-members:
 
-.. automodule:: borg.locking
+.. automodule:: borg.archive
     :members:
     :undoc-members:
 
-.. automodule:: borg.hash_sizes
+.. automodule:: borg.fuse
     :members:
     :undoc-members:
 
-.. automodule:: borg.logger
+.. automodule:: borg.platform
     :members:
     :undoc-members:
 
-.. automodule:: borg.remote
+.. automodule:: borg.locking
     :members:
     :undoc-members:
 
-.. automodule:: borg.fuse
+.. automodule:: borg.shellpattern
     :members:
     :undoc-members:
 
-.. automodule:: borg.archive
+.. automodule:: borg.repository
     :members:
     :undoc-members:
 
-.. automodule:: borg.helpers
+.. automodule:: borg.lrucache
     :members:
     :undoc-members:
 
-.. automodule:: borg.lrucache
+.. automodule:: borg.remote
     :members:
     :undoc-members:
 
-.. automodule:: borg.xattr
+.. automodule:: borg.hash_sizes
     :members:
     :undoc-members:
 
-.. automodule:: borg.archiver
+.. automodule:: borg.xattr
     :members:
     :undoc-members:
 
-.. automodule:: borg.repository
+.. automodule:: borg.helpers
     :members:
     :undoc-members:
 
@@ -62,7 +62,7 @@ API Documentation
     :members:
     :undoc-members:
 
-.. automodule:: borg.upgrader
+.. automodule:: borg.logger
     :members:
     :undoc-members:
 
@@ -70,15 +70,15 @@ API Documentation
     :members:
     :undoc-members:
 
-.. automodule:: borg.compress
+.. automodule:: borg.platform_linux
     :members:
     :undoc-members:
 
-.. automodule:: borg.platform_linux
+.. automodule:: borg.hashindex
     :members:
     :undoc-members:
 
-.. automodule:: borg.crypto
+.. automodule:: borg.compress
     :members:
     :undoc-members:
 
@@ -86,10 +86,10 @@ API Documentation
     :members:
     :undoc-members:
 
-.. automodule:: borg.platform_freebsd
+.. automodule:: borg.crypto
     :members:
     :undoc-members:
 
-.. automodule:: borg.hashindex
+.. automodule:: borg.platform_freebsd
     :members:
     :undoc-members:

docs/changes.rst

@@ -74,22 +74,34 @@ Other changes:
   - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945
 
 
-Version 1.0.3 (not released yet)
---------------------------------
+Version 1.0.3
+-------------
 
 Bug fixes:
 
-- prune: ignore checkpoints, #997
-- prune: fix bad validator, #942
-- fix capabilities extraction on Linux (set xattrs last, after chown())
+- prune: avoid that checkpoints are kept and completed archives are deleted in
+  a prune run), #997
+- prune: fix commandline argument validation - some valid command lines were
+  considered invalid (annoying, but harmless), #942
+- fix capabilities extraction on Linux (set xattrs last, after chown()), #1069
+- repository: fix commit tags being seen in data
+- when probing key files, do binary reads. avoids crash when non-borg binary
+  files are located in borg's key files directory.
+- handle SIGTERM and make a clean exit - avoids orphan lock files.
+- repository cache: don't cache large objects (avoid using lots of temp. disk
+  space), #1063
 
 Other changes:
 
-- update readthedocs URLs, #991
-- add missing docs for "borg break-lock", #992
-- borg create help: add some words to about the archive name
-- borg create help: document format tags, #894
 - Vagrantfile: OS X: update osxfuse / install lzma package, #933
+- setup.py: add check for platform_darwin.c
+- setup.py: on freebsd, use a llfuse release that builds ok
+- docs / help:
+
+  - update readthedocs URLs, #991
+  - add missing docs for "borg break-lock", #992
+  - borg create help: add some words to about the archive name
+  - borg create help: document format tags, #894
 
 
 Version 1.0.2

docs/usage/create.rst.inc

@@ -74,6 +74,12 @@ This command creates a backup archive containing all files found while recursive
 traversing all paths specified. The archive will consume almost no disk space for
 files or parts of files that have already been stored in other archives.
 
+The archive name needs to be unique. It must not end in '.checkpoint' or
+'.checkpoint.N' (with N being a number), because these names are used for
+checkpoints and treated in special ways.
+
+In the archive name, you may use the following format tags:
+{now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}
 
 To speed up pulling backups over sshfs and similar network file systems which do
 not provide correct inode information the --ignore-inode flag can be used. This

setup.py

@@ -26,12 +26,18 @@ install_requires = ['msgpack-python>=0.4.6', ]
 extras_require = {
     # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
     # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
+    # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
     # llfuse 2.0 will break API
     'fuse': ['llfuse<2.0', ],
 }
 
+if sys.platform.startswith('freebsd'):
+    # while llfuse 1.0 is the latest llfuse release right now,
+    # llfuse 0.41.1 is the latest release that actually builds on freebsd:
+    extras_require['fuse'] = ['llfuse==0.41.1', ]
+
 from setuptools import setup, Extension
 from setuptools.command.sdist import sdist
 
@@ -84,7 +90,7 @@ except ImportError:
     from distutils.command.build_ext import build_ext
     if not on_rtd and not all(os.path.exists(path) for path in [
         compress_source, crypto_source, chunker_source, hashindex_source,
-        platform_posix_source, platform_linux_source, platform_freebsd_source]):
+        platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')