Merge pull request #1991 from ThomasWaldmann/fix-pytest-xdist

use pytest-xdist to parallelize testing

borg/repository.py

@@ -3,15 +3,16 @@ from binascii import unhexlify
 from datetime import datetime
 from itertools import islice
 import errno
-import logging
-logger = logging.getLogger(__name__)
-
 import os
 import shutil
 import struct
 from zlib import crc32
 
 import msgpack
+
+from .logger import create_logger
+logger = create_logger()
+
 from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent, bin_to_hex
 from .hashindex import NSIndex
 from .locking import Lock, LockError, LockErrorT

borg/testsuite/archive.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from datetime import datetime, timezone
 from unittest.mock import Mock
 
@@ -131,11 +132,15 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized):
     assert not valid_msgpacked_dict(packed, item_keys_serialized)
 
 
+# pytest-xdist requires always same order for the keys and dicts:
+IK = sorted(list(ITEM_KEYS))
+
+
 @pytest.mark.parametrize('packed',
     [msgpack.packb(o) for o in [
         {b'path': b'/a/b/c'},  # small (different msgpack mapping type!)
-        dict((k, b'') for k in ITEM_KEYS),  # as big (key count) as it gets
-        dict((k, b'x' * 1000) for k in ITEM_KEYS),  # as big (key count and volume) as it gets
+        OrderedDict((k, b'') for k in IK),  # as big (key count) as it gets
+        OrderedDict((k, b'x' * 1000) for k in IK),  # as big (key count and volume) as it gets
     ]])
 def test_valid_msgpacked_items(packed, item_keys_serialized):
     assert valid_msgpacked_dict(packed, item_keys_serialized)

borg/testsuite/repository.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import shutil
 import sys
@@ -440,6 +441,8 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
 
         assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve']
         args = MockArgs()
+        # XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
+        logging.getLogger().setLevel(logging.INFO)
         # note: test logger is on info log level, so --info gets added automagically
         assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info']
         args.remote_path = 'borg-0.28.2'

borg/upgrader.py

@@ -1,10 +1,11 @@
 import datetime
-import logging
-logger = logging.getLogger(__name__)
 import os
 import shutil
 import time
 
+from .logger import create_logger
+logger = create_logger()
+
 from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent, bin_to_hex
 from .locking import Lock
 from .repository import Repository, MAGIC

requirements.d/development.txt

@@ -1,6 +1,7 @@
 virtualenv<14.0
 tox
 pytest
+pytest-xdist
 pytest-cov
 pytest-benchmark
 Cython

tox.ini

@@ -12,7 +12,7 @@ deps =
      -rrequirements.d/development.txt
      -rrequirements.d/attic.txt
      -rrequirements.d/fuse.txt
-commands = py.test -rs --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
+commands = py.test -n 8 -rs --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *