Merge pull request #2717 from ThomasWaldmann/backports3

backports

README.rst

@@ -155,7 +155,11 @@ see ``docs/suppport.rst`` in the source distribution).
 
 .. start-badges
 
-|doc| |build| |coverage| |bestpractices|
+|doc| |build| |coverage| |bestpractices| |bounties|
+
+.. |bounties| image:: https://api.bountysource.com/badge/team?team_id=78284&style=bounties_posted
+        :alt: Bounty Source
+        :target: https://www.bountysource.com/teams/borgbackup
 
 .. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=stable
         :alt: Documentation

borg/cache.py

@@ -392,6 +392,11 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
             except:
                 pass
 
+        # The cache can be used by a command that e.g. only checks against Manifest.Operation.WRITE,
+        # which does not have to include all flags from Manifest.Operation.READ.
+        # Since the sync will attempt to read archives, check compatibility with Manifest.Operation.READ.
+        self.manifest.check_repository_compatibility((Manifest.Operation.READ, ))
+
         self.begin_txn()
         with cache_if_remote(self.repository) as repository:
             legacy_cleanup()

borg/helpers.py

@@ -1085,7 +1085,12 @@ class Location:
     def to_key_filename(self):
         name = re.sub('[^\w]', '_', self.path).strip('_')
         if self.proto != 'file':
-            name = self.host + '__' + name
+            name = re.sub('[^\w]', '_', self.host) + '__' + name
+        if len(name) > 100:
+            # Limit file names to some reasonable length. Most file systems
+            # limit them to 255 [unit of choice]; due to variations in unicode
+            # handling we truncate to 100 *characters*.
+            name = name[:100]
         return os.path.join(get_keys_dir(), name)
 
     def __repr__(self):

borg/remote.py

@@ -31,6 +31,8 @@ MAX_INFLIGHT = 100
 
 def os_write(fd, data):
     """os.write wrapper so we do not lose data for partial writes."""
+    # TODO: this issue is fixed in cygwin since at least 2.8.0, remove this
+    #       wrapper / workaround when this version is considered ancient.
     # This is happening frequently on cygwin due to its small pipe buffer size of only 64kiB
     # and also due to its different blocking pipe behaviour compared to Linux/*BSD.
     # Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a

borg/testsuite/archiver.py

@@ -938,6 +938,12 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.add_unknown_feature(Manifest.Operation.WRITE)
         self.cmd_raises_unknown_feature(['create', self.repository_location + '::test', 'input'])
 
+    def test_unknown_feature_on_cache_sync(self):
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('delete', '--cache-only', self.repository_location)
+        self.add_unknown_feature(Manifest.Operation.READ)
+        self.cmd_raises_unknown_feature(['create', self.repository_location + '::test', 'input'])
+
     def test_unknown_feature_on_change_passphrase(self):
         print(self.cmd('init', self.repository_location))
         self.add_unknown_feature(Manifest.Operation.CHECK)
@@ -1786,7 +1792,6 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
         assert not self.cmd('list', self.repository_location)
 
 
-@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteArchiverTestCase(ArchiverTestCase):
     prefix = '__testsuite__:'
 

borg/testsuite/helpers.py

@@ -34,10 +34,19 @@ class BigIntTestCase(BaseTestCase):
 
 
 class TestLocationWithoutEnv:
-    def test_ssh(self, monkeypatch):
+    @pytest.fixture
+    def keys_dir(self, tmpdir, monkeypatch):
+        tmpdir = str(tmpdir)
+        monkeypatch.setenv('BORG_KEYS_DIR', tmpdir)
+        if not tmpdir.endswith(os.path.sep):
+            tmpdir += os.path.sep
+        return tmpdir
+
+    def test_ssh(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('ssh://user@host:1234/some/path::archive')) == \
             "Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive='archive')"
+        assert Location('ssh://user@host:1234/some/path::archive').to_key_filename() == keys_dir + 'host__some_path'
         assert repr(Location('ssh://user@host:1234/some/path')) == \
             "Location(proto='ssh', user='user', host='host', port=1234, path='/some/path', archive=None)"
         assert repr(Location('ssh://user@host/some/path')) == \
@@ -46,12 +55,14 @@ class TestLocationWithoutEnv:
             "Location(proto='ssh', user='user', host='::', port=1234, path='/some/path', archive='archive')"
         assert repr(Location('ssh://user@[::]:1234/some/path')) == \
             "Location(proto='ssh', user='user', host='::', port=1234, path='/some/path', archive=None)"
+        assert Location('ssh://user@[::]:1234/some/path').to_key_filename() == keys_dir + '____some_path'
         assert repr(Location('ssh://user@[::]/some/path')) == \
             "Location(proto='ssh', user='user', host='::', port=None, path='/some/path', archive=None)"
         assert repr(Location('ssh://user@[2001:db8::]:1234/some/path::archive')) == \
             "Location(proto='ssh', user='user', host='2001:db8::', port=1234, path='/some/path', archive='archive')"
         assert repr(Location('ssh://user@[2001:db8::]:1234/some/path')) == \
             "Location(proto='ssh', user='user', host='2001:db8::', port=1234, path='/some/path', archive=None)"
+        assert Location('ssh://user@[2001:db8::]:1234/some/path').to_key_filename() == keys_dir + '2001_db8____some_path'
         assert repr(Location('ssh://user@[2001:db8::]/some/path')) == \
             "Location(proto='ssh', user='user', host='2001:db8::', port=None, path='/some/path', archive=None)"
         assert repr(Location('ssh://user@[2001:db8::c0:ffee]:1234/some/path::archive')) == \
@@ -66,15 +77,17 @@ class TestLocationWithoutEnv:
             "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=1234, path='/some/path', archive=None)"
         assert repr(Location('ssh://user@[2001:db8::192.0.2.1]/some/path')) == \
             "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='/some/path', archive=None)"
+        assert Location('ssh://user@[2001:db8::192.0.2.1]/some/path').to_key_filename() == keys_dir + '2001_db8__192_0_2_1__some_path'
 
-    def test_file(self, monkeypatch):
+    def test_file(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('file:///some/path::archive')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive='archive')"
         assert repr(Location('file:///some/path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/path', archive=None)"
+        assert Location('file:///some/path').to_key_filename() == keys_dir + 'some_path'
 
-    def test_scp(self, monkeypatch):
+    def test_scp(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('user@host:/some/path::archive')) == \
             "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive='archive')"
@@ -96,42 +109,55 @@ class TestLocationWithoutEnv:
             "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='/some/path', archive='archive')"
         assert repr(Location('user@[2001:db8::192.0.2.1]:/some/path')) == \
             "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='/some/path', archive=None)"
+        assert Location('user@[2001:db8::192.0.2.1]:/some/path').to_key_filename() == keys_dir + '2001_db8__192_0_2_1__some_path'
 
-    def test_smb(self, monkeypatch):
+    def test_smb(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('file:////server/share/path::archive')) == \
             "Location(proto='file', user=None, host=None, port=None, path='//server/share/path', archive='archive')"
+        assert Location('file:////server/share/path::archive').to_key_filename() == keys_dir + 'server_share_path'
 
-    def test_folder(self, monkeypatch):
+    def test_folder(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('path::archive')) == \
             "Location(proto='file', user=None, host=None, port=None, path='path', archive='archive')"
         assert repr(Location('path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='path', archive=None)"
+        assert Location('path').to_key_filename() == keys_dir + 'path'
 
-    def test_abspath(self, monkeypatch):
+    def test_long_path(self, monkeypatch, keys_dir):
+        monkeypatch.delenv('BORG_REPO', raising=False)
+        assert Location(os.path.join(*(40 * ['path']))).to_key_filename() == keys_dir + '_'.join(20 * ['path']) + '_'
+
+    def test_abspath(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('/some/absolute/path::archive')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')"
         assert repr(Location('/some/absolute/path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)"
+        assert Location('/some/absolute/path').to_key_filename() == keys_dir + 'some_absolute_path'
         assert repr(Location('ssh://user@host/some/path')) == \
                "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)"
+        assert Location('ssh://user@host/some/path').to_key_filename() == keys_dir + 'host__some_path'
 
-    def test_relpath(self, monkeypatch):
+    def test_relpath(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('some/relative/path::archive')) == \
             "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')"
         assert repr(Location('some/relative/path')) == \
             "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
+        assert Location('some/relative/path').to_key_filename() == keys_dir + 'some_relative_path'
         assert repr(Location('ssh://user@host/./some/path')) == \
                "Location(proto='ssh', user='user', host='host', port=None, path='/./some/path', archive=None)"
+        assert Location('ssh://user@host/./some/path').to_key_filename() == keys_dir + 'host__some_path'
         assert repr(Location('ssh://user@host/~/some/path')) == \
                "Location(proto='ssh', user='user', host='host', port=None, path='/~/some/path', archive=None)"
+        assert Location('ssh://user@host/~/some/path').to_key_filename() == keys_dir + 'host__some_path'
         assert repr(Location('ssh://user@host/~user/some/path')) == \
                "Location(proto='ssh', user='user', host='host', port=None, path='/~user/some/path', archive=None)"
+        assert Location('ssh://user@host/~user/some/path').to_key_filename() == keys_dir + 'host__user_some_path'
 
-    def test_with_colons(self, monkeypatch):
+    def test_with_colons(self, monkeypatch, keys_dir):
         monkeypatch.delenv('BORG_REPO', raising=False)
         assert repr(Location('/abs/path:w:cols::arch:col')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols', archive='arch:col')"
@@ -139,6 +165,7 @@ class TestLocationWithoutEnv:
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive='archive')"
         assert repr(Location('/abs/path:with:colons')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)"
+        assert Location('/abs/path:with:colons').to_key_filename() == keys_dir + 'abs_path_with_colons'
 
     def test_user_parsing(self):
         # see issue #1930

borg/testsuite/repository.py

@@ -416,7 +416,6 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
             self.assert_equal(self.repository.get(H(0)), b'data2')
 
 
-@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteRepositoryTestCase(RepositoryTestCase):
 
     def open(self, create=False):
@@ -449,7 +448,6 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
         assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--umask=077', '--info']
 
 
-@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
 
     def open(self, create=False):

docs/changes.rst

@@ -1,7 +1,7 @@
 Important notes
 ===============
 
-This section is used for infos about security and corruption issues.
+This section provides information about security and corruption issues.
 
 .. _tam_vuln:
 

docs/misc/prune-example.txt

@@ -2,7 +2,7 @@ borg prune visualized
 =====================
 
 Assume it is 2016-01-01, today's backup has not yet been made and you have
-created at least one backup on each day in 2015 except on 2015-12-20 (no
+created at least one backup on each day in 2015 except on 2015-12-19 (no
 backup made on that day).
 
 This is what borg prune --keep-daily 14 --keep-monthly 6 would keep.
@@ -45,7 +45,7 @@ Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
 Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su  Mo Tu We Th Fr Sa Su
           1  2  3  4                     1      1  2  3  4  5  6
  5  6  7  8  9 10 11   2  3  4  5  6  7  8   7  8  9 10 11 12 13
-12 13 14 15 16 17 18   9 10 11 12 13 14 15  14 15 16 17d18d19d20
+12 13 14 15 16 17 18   9 10 11 12 13 14 15  14 15 16 17d18d19 20d
 19 20 21 22 23 24 25  16 17 18 19 20 21 22  21d22d23d24d25d26d27d
 26 27 28 29 30 31m    23 24 25 26 27 28 29  28d29d30d31d
                       30m
@@ -66,8 +66,8 @@ List view
  9. 2015-12-23
 10. 2015-12-22
 11. 2015-12-21
-    (no backup made on 2015-12-20)
-12. 2015-12-19
+12. 2015-12-20
+    (no backup made on 2015-12-19)
 13. 2015-12-18
 14. 2015-12-17
 
@@ -83,7 +83,7 @@ Jun. December is not considered for this rule, because that backup was already
 kept because of the daily rule.
 
 2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was
-made on 2015-12-20. If a backup had been made on that day, it would not keep
+made on 2015-12-19. If a backup had been made on that day, it would not keep
 the one from 2015-12-17.
 
 We did not include yearly, weekly, hourly, minutely or secondly rules to keep