Merge pull request #270 from ThomasWaldmann/benchmark

integrate pytest-benchmark, test create, extract, list, delete, info,…

.travis/run.sh

@@ -17,7 +17,7 @@ source ~/.venv/bin/activate
 
 if [[ "$(uname -s)" == "Darwin" ]]; then
     # no fakeroot on OS X
-    sudo tox -e $TOXENV
+    sudo tox -e $TOXENV -r
 else
-    fakeroot -u tox
+    fakeroot -u tox -r
 fi

borg/testsuite/archiver.py

@@ -71,6 +71,29 @@ class environment_variable:
                 os.environ[k] = v
 
 
+def exec_cmd(*args, archiver=None, fork=False, **kw):
+    if fork:
+        try:
+            borg = (sys.executable, '-m', 'borg.archiver')
+            output = subprocess.check_output(borg + args)
+            ret = 0
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            ret = e.returncode
+        return ret, os.fsdecode(output)
+    else:
+        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
+        try:
+            sys.stdin = StringIO()
+            sys.stdout = sys.stderr = output = StringIO()
+            if archiver is None:
+                archiver = Archiver()
+            ret = archiver.run(list(args))
+            return ret, output.getvalue()
+        finally:
+            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+
+
 class ArchiverTestCaseBase(BaseTestCase):
 
     prefix = ''
@@ -102,34 +125,12 @@ class ArchiverTestCaseBase(BaseTestCase):
         shutil.rmtree(self.tmpdir)
 
     def cmd(self, *args, **kw):
-        exit_code = kw.get('exit_code', 0)
-        fork = kw.get('fork', False)
-        if fork:
-            try:
-                output = subprocess.check_output((sys.executable, '-m', 'borg.archiver') + args)
-                ret = 0
-            except subprocess.CalledProcessError as e:
-                output = e.output
-                ret = e.returncode
-            output = os.fsdecode(output)
-            if ret != exit_code:
-                print(output)
-            self.assert_equal(exit_code, ret)
-            return output
-        args = list(args)
-        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
-        try:
-            sys.stdin = StringIO()
-            output = StringIO()
-            sys.stdout = sys.stderr = output
-            ret = self.archiver.run(args)
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
-            if ret != exit_code:
-                print(output.getvalue())
-            self.assert_equal(exit_code, ret)
-            return output.getvalue()
-        finally:
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+        exit_code = kw.pop('exit_code', 0)
+        ret, output = exec_cmd(*args, archiver=self.archiver, **kw)
+        if ret != exit_code:
+            print(output)
+        self.assert_equal(ret, exit_code)
+        return output
 
     def create_src_archive(self, name):
         self.cmd('create', self.repository_location + '::' + name, src_dir)

borg/testsuite/benchmark.py

@@ -0,0 +1,106 @@
+"""
+Do benchmarks using pytest-benchmark.
+
+Usage:
+
+    py.test --benchmark-only
+"""
+
+import os
+
+import pytest
+
+from .archiver import changedir, exec_cmd
+
+
+# TODO: use fixture params to test python code and binary
+@pytest.fixture
+def cmd():
+    return exec_cmd
+
+
+@pytest.yield_fixture
+def repo_url(request, tmpdir):
+    os.environ['BORG_PASSPHRASE'] = '123456'
+    os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
+    os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1'
+    os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys'))
+    os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache'))
+    yield str(tmpdir.join('repository'))
+    tmpdir.remove(rec=1)
+
+
+@pytest.fixture(params=["none", "passphrase"])
+def repo(request, cmd, repo_url):
+    cmd('init', '--encryption', request.param, repo_url)
+    return repo_url
+
+
+@pytest.yield_fixture(scope='session', params=["zeros", "random"])
+def testdata(request, tmpdir_factory):
+    count, size = 10, 1000*1000
+    p = tmpdir_factory.mktemp('data')
+    data_type = request.param
+    if data_type == 'zeros':
+        # do not use a binary zero (\0) to avoid sparse detection
+        data = lambda: b'0' * size
+    if data_type == 'random':
+        rnd = open('/dev/urandom', 'rb')
+        data = lambda: rnd.read(size)
+    for i in range(count):
+        with open(str(p.join(str(i))), "wb") as f:
+            f.write(data())
+    if data_type == 'random':
+        rnd.close()
+    yield str(p)
+    p.remove(rec=1)
+
+
+@pytest.fixture(params=['none', 'lz4'])
+def archive(request, cmd, repo, testdata):
+    archive_url = repo + '::test'
+    cmd('create', '--compression', request.param, archive_url, testdata)
+    return archive_url
+
+
+def test_create_none(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'none', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_create_lz4(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'lz4', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_extract(benchmark, cmd, archive, tmpdir):
+    with changedir(str(tmpdir)):
+        result, out = benchmark.pedantic(cmd, ('extract', archive))
+    assert result == 0
+
+
+def test_delete(benchmark, cmd, archive):
+    result, out = benchmark.pedantic(cmd, ('delete', archive))
+    assert result == 0
+
+
+def test_list(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'list', archive)
+    assert result == 0
+
+
+def test_info(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'info', archive)
+    assert result == 0
+
+
+def test_check(benchmark, cmd, archive):
+    repo = archive.split('::')[0]
+    result, out = benchmark(cmd, 'check', repo)
+    assert result == 0
+
+
+def test_help(benchmark, cmd):
+    result, out = benchmark(cmd, 'help')
+    assert result == 0
+

requirements.d/development.txt

@@ -2,4 +2,5 @@ tox
 mock
 pytest
 pytest-cov<2.0.0
+pytest-benchmark==3.0.0b1
 Cython