Merge pull request #2719 from ThomasWaldmann/backports4

More backports

borg/archiver.py

@@ -486,7 +486,7 @@ class Archiver:
                 else:
                     if stat.S_ISDIR(item[b'mode']):
                         dirs.append(item)
-                        archive.extract_item(item, restore_attrs=False)
+                        archive.extract_item(item, stdout=stdout, restore_attrs=False)
                     else:
                         archive.extract_item(item, stdout=stdout, sparse=sparse)
             except BackupOSError as e:
@@ -496,7 +496,7 @@ class Archiver:
             while dirs:
                 dir_item = dirs.pop(-1)
                 try:
-                    archive.extract_item(dir_item)
+                    archive.extract_item(dir_item, stdout=stdout)
                 except BackupOSError as e:
                     self.print_warning('%s: %s', remove_surrogates(dir_item[b'path']), e)
         for pattern in include_patterns:

borg/fuse.py

@@ -60,7 +60,10 @@ class FuseOperations(llfuse.Operations):
         self.items = {}
         self.parent = {}
         self.contents = defaultdict(dict)
-        self.default_dir = {b'mode': 0o40755, b'mtime': int(time.time() * 1e9), b'uid': os.getuid(), b'gid': os.getgid()}
+        self.default_uid = os.getuid()
+        self.default_gid = os.getgid()
+        self.default_dir = {b'mode': 0o40755, b'mtime': int(time.time() * 1e9),
+                            b'uid': self.default_uid, b'gid': self.default_gid}
         self.pending_archives = {}
         self.accounted_chunks = {}
         self.cache = ItemCache()
@@ -209,8 +212,8 @@ class FuseOperations(llfuse.Operations):
         entry.attr_timeout = 300
         entry.st_mode = item[b'mode']
         entry.st_nlink = item.get(b'nlink', 1)
-        entry.st_uid = item[b'uid']
-        entry.st_gid = item[b'gid']
+        entry.st_uid = item[b'uid'] if item[b'uid'] >= 0 else self.default_uid
+        entry.st_gid = item[b'gid'] if item[b'gid'] >= 0 else self.default_gid
         entry.st_rdev = item.get(b'rdev', 0)
         entry.st_size = size
         entry.st_blksize = 512

borg/testsuite/upgrader.py

@@ -1,14 +1,8 @@
 import os
+import tarfile
 
 import pytest
 
-try:
-    import attic.repository
-    import attic.key
-    import attic.helpers
-except ImportError:
-    attic = None
-
 from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
 from ..helpers import get_keys_dir
 from ..key import KeyfileKey
@@ -16,6 +10,28 @@ from ..archiver import UMASK_DEFAULT
 from ..repository import Repository
 
 
+# tar with a repo and repo keyfile from attic
+ATTIC_TAR = os.path.join(os.path.dirname(__file__), 'attic.tar.gz')
+
+
+def untar(tarfname, path, what):
+    """
+    extract <tarfname> tar archive to <path>, all stuff starting with <what>.
+
+    return path to <what>.
+    """
+
+    def files(members):
+        for tarinfo in members:
+            if tarinfo.name.startswith(what):
+                yield tarinfo
+
+    with tarfile.open(tarfname, 'r') as tf:
+        tf.extractall(path, members=files(tf))
+
+    return os.path.join(path, what)
+
+
 def repo_valid(path):
     """
     utility function to check if borg can open a repository
@@ -47,15 +63,10 @@ def attic_repo(tmpdir):
     create an attic repo with some stuff in it
 
     :param tmpdir: path to the repository to be created
-    :returns: a attic.repository.Repository object
+    :returns: path to attic repository
     """
-    attic_repo = attic.repository.Repository(str(tmpdir), create=True)
-    # throw some stuff in that repo, copied from `RepositoryTestCase.test1`
-    for x in range(100):
-        attic_repo.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
-    attic_repo.commit()
-    attic_repo.close()
-    return attic_repo
+    # there is some stuff in that repo, copied from `RepositoryTestCase.test1`
+    return untar(ATTIC_TAR, str(tmpdir), 'repo')
 
 
 @pytest.fixture(params=[True, False])
@@ -63,52 +74,36 @@ def inplace(request):
     return request.param
 
 
-@pytest.mark.skipif(attic is None, reason='cannot find an attic install')
-def test_convert_segments(tmpdir, attic_repo, inplace):
+def test_convert_segments(attic_repo, inplace):
     """test segment conversion
 
     this will load the given attic repository, list all the segments
     then convert them one at a time. we need to close the repo before
     conversion otherwise we have errors from borg
 
-    :param tmpdir: a temporary directory to run the test in (builtin
-    fixture)
     :param attic_repo: a populated attic repository (fixture)
     """
+    repo_path = attic_repo
     # check should fail because of magic number
-    assert not repo_valid(tmpdir)
-    repository = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    assert not repo_valid(repo_path)
+    repository = AtticRepositoryUpgrader(repo_path, create=False)
     with repository:
         segments = [filename for i, filename in repository.io.segment_iterator()]
     repository.convert_segments(segments, dryrun=False, inplace=inplace)
     repository.convert_cache(dryrun=False)
-    assert repo_valid(tmpdir)
-
-
-class MockArgs:
-    """
-    mock attic location
-
-    this is used to simulate a key location with a properly loaded
-    repository object to create a key file
-    """
-    def __init__(self, path):
-        self.repository = attic.helpers.Location(path)
+    assert repo_valid(repo_path)
 
 
 @pytest.fixture()
-def attic_key_file(attic_repo, tmpdir, monkeypatch):
+def attic_key_file(tmpdir, monkeypatch):
     """
     create an attic key file from the given repo, in the keys
     subdirectory of the given tmpdir
 
-    :param attic_repo: an attic.repository.Repository object (fixture
-    define above)
     :param tmpdir: a temporary directory (a builtin fixture)
-    :returns: the KeyfileKey object as returned by
-    attic.key.KeyfileKey.create()
+    :returns: path to key file
     """
-    keys_dir = str(tmpdir.mkdir('keys'))
+    keys_dir = untar(ATTIC_TAR, str(tmpdir), 'keys')
 
     # we use the repo dir for the created keyfile, because we do
     # not want to clutter existing keyfiles
@@ -119,44 +114,42 @@ def attic_key_file(attic_repo, tmpdir, monkeypatch):
     # about anyways. in real runs, the original key will be retained.
     monkeypatch.setenv('BORG_KEYS_DIR', keys_dir)
     monkeypatch.setenv('ATTIC_PASSPHRASE', 'test')
-    return attic.key.KeyfileKey.create(attic_repo,
-                                       MockArgs(keys_dir))
+
+    return os.path.join(keys_dir, 'repo')
 
 
-@pytest.mark.skipif(attic is None, reason='cannot find an attic install')
-def test_keys(tmpdir, attic_repo, attic_key_file):
+def test_keys(attic_repo, attic_key_file):
     """test key conversion
 
     test that we can convert the given key to a properly formatted
     borg key. assumes that the ATTIC_KEYS_DIR and BORG_KEYS_DIR have
     been properly populated by the attic_key_file fixture.
 
-    :param tmpdir: a temporary directory (a builtin fixture)
-    :param attic_repo: an attic.repository.Repository object (fixture
-    define above)
-    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    :param attic_repo: path to an attic repository (fixture defined above)
+    :param attic_key_file: path to an attic key file (fixture defined above)
     """
-    with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository:
+    keyfile_path = attic_key_file
+    assert not key_valid(keyfile_path)  # not upgraded yet
+    with AtticRepositoryUpgrader(attic_repo, create=False) as repository:
         keyfile = AtticKeyfileKey.find_key_file(repository)
         AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False)
-    assert key_valid(attic_key_file.path)
+    assert key_valid(keyfile_path)
 
 
-@pytest.mark.skipif(attic is None, reason='cannot find an attic install')
-def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace):
+def test_convert_all(attic_repo, attic_key_file, inplace):
     """test all conversion steps
 
     this runs everything. mostly redundant test, since everything is
     done above. yet we expect a NotImplementedError because we do not
     convert caches yet.
 
-    :param tmpdir: a temporary directory (a builtin fixture)
-    :param attic_repo: an attic.repository.Repository object (fixture
-    define above)
-    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    :param attic_repo: path to an attic repository (fixture defined above)
+    :param attic_key_file: path to an attic key file (fixture defined above)
     """
+    repo_path = attic_repo
+
     # check should fail because of magic number
-    assert not repo_valid(tmpdir)
+    assert not repo_valid(repo_path)
 
     def stat_segment(path):
         return os.stat(os.path.join(path, 'data', '0', '0'))
@@ -164,8 +157,8 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace):
     def first_inode(path):
         return stat_segment(path).st_ino
 
-    orig_inode = first_inode(attic_repo.path)
-    with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository:
+    orig_inode = first_inode(repo_path)
+    with AtticRepositoryUpgrader(repo_path, create=False) as repository:
         # replicate command dispatch, partly
         os.umask(UMASK_DEFAULT)
         backup = repository.upgrade(dryrun=False, inplace=inplace)
@@ -179,8 +172,8 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace):
             # permissions, which is wrong
             assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0
 
-    assert key_valid(attic_key_file.path)
-    assert repo_valid(tmpdir)
+    assert key_valid(attic_key_file)
+    assert repo_valid(repo_path)
 
 
 def test_hardlink(tmpdir, inplace):

requirements.d/attic.txt

@@ -1,5 +0,0 @@
-# Please note:
-# attic only builds using OpenSSL 1.0.x, it can not be installed using OpenSSL >= 1.1.0.
-# If attic is not installed, our unit tests will just skip the tests that require attic.
-attic
-

tox.ini

@@ -10,7 +10,6 @@ envlist = py{34,35,36},flake8
 changedir = {toxworkdir}
 deps =
      -rrequirements.d/development.txt
-     -rrequirements.d/attic.txt
      -rrequirements.d/fuse.txt
 commands = py.test -n {env:XDISTN:4} -rs --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars: