Merge pull request #5397 from ThomasWaldmann/modernize-1.1

modernize 1.1

.travis.yml

@@ -7,37 +7,29 @@ cache:
 matrix:
     fast_finish: true
     include:
-        - python: "3.4"
+        - python: "3.5"
           os: linux
-          dist: trusty
-          env: TOXENV=py34
-        - python: "3.5.2"  # Use version 3.5.2 because it has lzma support while 3.5(.0) does not
-          os: linux
-          dist: trusty
+          dist: xenial
           env: TOXENV=py35
         - python: "3.6"
           os: linux
-          dist: trusty
+          dist: bionic
           env: TOXENV=py36
         - python: "3.7"
           os: linux
-          dist: xenial
+          dist: bionic
           env: TOXENV=py37
         - python: "3.8"
           os: linux
-          dist: xenial
+          dist: focal
           env: TOXENV=py38
-        - python: "3.9-dev"
-          os: linux
-          dist: xenial
-          env: TOXENV=py39
         - python: "3.9-dev"
           os: linux
           dist: focal
           env: TOXENV=py39
-        - python: "3.6"
+        - python: "3.8"
           os: linux
-          dist: xenial
+          dist: focal
           env: TOXENV=flake8
         - language: generic
           os: osx

Vagrantfile

@@ -18,21 +18,11 @@ def packages_debianoid(user)
     usermod -a -G fuse #{user}
     chgrp fuse /dev/fuse
     chmod 666 /dev/fuse
-    apt-get install -y fakeroot build-essential git
+    apt-get install -y fakeroot build-essential git curl
     apt-get install -y python3-dev python3-setuptools
     # for building python:
     apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
-    # this way it works on older dists (like ubuntu 12.04) also:
-    # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as
-    # newer versions are not compatible with py 3.2 any more.
-    if which easy_install3 2> /dev/null; then
-        # works up to xenial / stretch
-        easy_install3 -i https://pypi.python.org/simple/ 'pip<8.0'
-        pip3 install 'virtualenv<14.0'
-    else
-        # works on recent debian / ubuntu
-        apt-get install -y python3-pip virtualenv python3-virtualenv
-    fi
+    apt-get install -y python3-pip virtualenv python3-virtualenv
   EOF
 end
 
@@ -229,10 +219,9 @@ def install_pythons(boxname)
     . ~/.bash_profile
     pyenv install 3.5.3  # tests, 3.5.3 is first to support openssl 1.1
     pyenv install 3.6.2  # tests
-    pyenv install 3.7.0  # tests
+    pyenv install 3.7.9  # binary build, use latest 3.7.x release
     pyenv install 3.8.0  # tests
     pyenv install 3.9.0  # tests
-    pyenv install 3.5.10  # binary build, use latest 3.5.x release
     pyenv rehash
   EOF
 end
@@ -249,9 +238,9 @@ def build_pyenv_venv(boxname)
   return <<-EOF
     . ~/.bash_profile
     cd /vagrant/borg
-    # use the latest 3.5 release
-    pyenv global 3.5.10
-    pyenv virtualenv 3.5.10 borg-env
+    # use the latest 3.7 release
+    pyenv global 3.7.9
+    pyenv virtualenv 3.7.9 borg-env
     ln -s ~/.pyenv/versions/borg-env .
   EOF
 end
@@ -288,7 +277,7 @@ def install_pyinstaller()
     . borg-env/bin/activate
     git clone https://github.com/thomaswaldmann/pyinstaller.git
     cd pyinstaller
-    git checkout v3.2.1
+    git checkout v4.0-maint
     python setup.py install
   EOF
 end
@@ -312,8 +301,8 @@ def run_tests(boxname)
     . ../borg-env/bin/activate
     if which pyenv 2> /dev/null; then
       # for testing, use the earliest point releases of the supported python versions:
-      pyenv global 3.5.3 3.6.2 3.7.0 3.8.0 3.9.0
-      pyenv local 3.5.3 3.6.2 3.7.0 3.8.0 3.9.0
+      pyenv global 3.5.3 3.6.2 3.7.9 3.8.0 3.9.0
+      pyenv local 3.5.3 3.6.2 3.7.9 3.8.0 3.9.0
     fi
     # otherwise: just use the system python
     if which fakeroot 2> /dev/null; then
@@ -403,18 +392,6 @@ Vagrant.configure(2) do |config|
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64")
   end
 
-  config.vm.define "trusty64" do |b|
-    b.vm.box = "ubuntu/trusty64"
-    b.vm.provider :virtualbox do |v|
-      v.memory = 1024 + $wmem
-    end
-    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
-    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64")
-  end
-
   config.vm.define "buster64" do |b|
     b.vm.box = "debian/buster64"
     b.vm.provider :virtualbox do |v|
@@ -434,41 +411,13 @@ Vagrant.configure(2) do |config|
     end
     b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("stretch64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
-  end
-
-  config.vm.define "jessie32" do |b|
-    b.vm.box = "debian8-i386"
-    b.vm.provider :virtualbox do |v|
-      v.memory = 768 + $wmem
-    end
-    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
-    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("jessie32")
-    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("jessie32")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("jessie32")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
-    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
-    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("jessie32")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie32")
-  end
-
-  config.vm.define "jessie64" do |b|
-    b.vm.box = "debian8-amd64"
-    b.vm.provider :virtualbox do |v|
-      v.memory = 1024 + $wmem
-    end
-    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
-    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("jessie64")
-    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("jessie64")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("jessie64")
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64")
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
     b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
-    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("jessie64")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
   end
 
   # OS X

docs/development.rst

@@ -182,7 +182,7 @@ Some more advanced examples::
   # verify a changed tox.ini (run this after any change to tox.ini):
   fakeroot -u tox --recreate
 
-  fakeroot -u tox -e py34  # run all tests, but only on python 3.4
+  fakeroot -u tox -e py37  # run all tests, but only on python 3.7
 
   fakeroot -u tox borg.testsuite.locking  # only run 1 test module
 
@@ -321,7 +321,7 @@ Checklist:
 - Render ``CHANGES.rst`` via ``make html`` and check for markup errors.
 - Verify that ``MANIFEST.in`` and ``setup.py`` are complete.
 - ``python setup.py build_usage ; python setup.py build_man`` and
-  commit (be sure to build with Python 3.4 or 3.5 as Python 3.6 added `more
+  commit (be sure to build with Python 3.5 as Python 3.6 added `more
   guaranteed hashing algorithms
   <https://github.com/borgbackup/borg/issues/2123>`_).
 - Tag the release::

docs/installation.rst

@@ -140,7 +140,7 @@ Dependencies
 To install |project_name| from a source package (including pip), you have to install the
 following dependencies first:
 
-* `Python 3`_ >= 3.4.0, plus development headers. Even though Python 3 is not
+* `Python 3`_ >= 3.5.0, plus development headers. Even though Python 3 is not
   the default Python version on most systems, it is usually available as an
   optional install.
 * OpenSSL_ >= 1.0.0, plus development headers.
@@ -237,7 +237,7 @@ and commands to make FUSE work for using the mount command.
 
      pkg install -y python3 openssl fusefs-libs pkgconf
      pkg install -y git
-     python3.4 -m ensurepip # to install pip for Python3
+     python3 -m ensurepip  # to install pip for Python3
      To use the mount command:
      echo 'fuse_load="YES"' >> /boot/loader.conf
      echo 'vfs.usermount=1' >> /etc/sysctl.conf
@@ -269,7 +269,7 @@ Use the Cygwin installer to install the dependencies::
 
 You can then install ``pip`` and ``virtualenv``::
 
-    easy_install-3.4 pip
+    easy_install-3.7 pip
     pip install virtualenv
 
 

requirements.d/development.lock.txt

@@ -1,13 +1,13 @@
-setuptools==40.8.0
-setuptools-scm==3.2.0
-pip==19.0.3
-virtualenv==16.4.3
-pluggy==0.9.0
-tox==3.7.0
-pytest==4.3.1
-pytest-xdist==1.27.0
-pytest-cov==2.6.1
-pytest-benchmark==3.2.2
-Cython==0.29.6
-twine==1.13.0
-readme-renderer<25.0
+setuptools==50.3.0
+setuptools-scm==4.1.2
+pip==20.2.3
+virtualenv==20.0.33
+importlib-metadata==1.7.0
+pluggy==0.13.1
+tox==3.20.0
+pytest==6.1.1
+pytest-xdist==2.1.0
+pytest-cov==2.10.1
+pytest-benchmark==3.2.3
+Cython==0.29.21
+twine==1.15.0

requirements.d/development.txt

@@ -2,13 +2,10 @@ setuptools
 setuptools_scm
 pip
 virtualenv
-# newer cryptography does not build in the jessie vagrant boxes
-cryptography<3.0
 tox
 pytest
 pytest-xdist
 pytest-cov
 pytest-benchmark
-Cython!=0.27
+Cython
 twine
-readme-renderer<25.0

setup.py

@@ -23,7 +23,7 @@ prefer_system_libb2 = True
 
 # prefer_system_msgpack is another option, but you need to set it in src/borg/helpers.py.
 
-min_python = (3, 4)
+min_python = (3, 5)
 my_python = sys.version_info
 
 if my_python < min_python:
@@ -40,16 +40,12 @@ install_requires = []
 # llfuse package. "borg mount" needs llfuse to work.
 # if you do not have llfuse, do not require it, most of borgbackup will work.
 extras_require = {
-    # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0
-    # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0
-    # llfuse 2.0 will break API
     'fuse': [
+        # note: the C code generated by cython and bundled into the pypi package
+        # is the reason for these requirements. Pre-1.3.7 it was made by a too old
+        # cython to work on py39. In 1.3.7 it was made by a cython version that
+        # produces py39 compatible code, but that also led to a freebsd compile issue.
+        # if you cythonize yourself, you can use either version.
         'llfuse >=1.3.4, <1.3.7; python_version <"3.9"',  # broken on py39
         'llfuse >=1.3.7, <2.0; python_version >="3.9"',  # broken on freebsd
     ],
@@ -857,7 +853,6 @@ setup(
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

src/borg/archiver.py

@@ -2390,11 +2390,11 @@ class Archiver:
 
         {now}
             The current local date and time, by default in ISO-8601 format.
-            You can also supply your own `format string <https://docs.python.org/3.4/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {now:%Y-%m-%d_%H:%M:%S}
+            You can also supply your own `format string <https://docs.python.org/3.7/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {now:%Y-%m-%d_%H:%M:%S}
 
         {utcnow}
             The current UTC date and time, by default in ISO-8601 format.
-            You can also supply your own `format string <https://docs.python.org/3.4/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {utcnow:%Y-%m-%d_%H:%M:%S}
+            You can also supply your own `format string <https://docs.python.org/3.7/library/datetime.html#strftime-and-strptime-behavior>`_, e.g. {utcnow:%Y-%m-%d_%H:%M:%S}
 
         {user}
             The user name (or UID, if no name is available) of the user running borg.

src/borg/helpers.py

@@ -28,6 +28,7 @@ from datetime import datetime, timezone, timedelta
 from functools import partial, lru_cache
 from itertools import islice
 from operator import attrgetter
+from os import scandir
 from string import Formatter
 from shutil import get_terminal_size
 
@@ -2134,68 +2135,6 @@ def consume(iterator, n=None):
         # advance to the empty slice starting at position n
         next(islice(iterator, n, n), None)
 
-# GenericDirEntry, scandir_generic (c) 2012 Ben Hoyt
-# from the python-scandir package (3-clause BSD license, just like us, so no troubles here)
-# note: simplified version
-
-
-class GenericDirEntry:
-    __slots__ = ('name', '_scandir_path', '_path')
-
-    def __init__(self, scandir_path, name):
-        self._scandir_path = scandir_path
-        self.name = name
-        self._path = None
-
-    @property
-    def path(self):
-        if self._path is None:
-            self._path = os.path.join(self._scandir_path, self.name)
-        return self._path
-
-    def stat(self, follow_symlinks=True):
-        assert not follow_symlinks
-        return os.stat(self.path, follow_symlinks=follow_symlinks)
-
-    def _check_type(self, type):
-        st = self.stat(False)
-        return stat.S_IFMT(st.st_mode) == type
-
-    def is_dir(self, follow_symlinks=True):
-        assert not follow_symlinks
-        return self._check_type(stat.S_IFDIR)
-
-    def is_file(self, follow_symlinks=True):
-        assert not follow_symlinks
-        return self._check_type(stat.S_IFREG)
-
-    def is_symlink(self):
-        return self._check_type(stat.S_IFLNK)
-
-    def inode(self):
-        st = self.stat(False)
-        return st.st_ino
-
-    def __repr__(self):
-        return '<{0}: {1!r}>'.format(self.__class__.__name__, self.path)
-
-
-def scandir_generic(path='.'):
-    """Like os.listdir(), but yield DirEntry objects instead of returning a list of names."""
-    for name in os.listdir(path):
-        yield GenericDirEntry(path, name)
-
-
-try:
-    from os import scandir
-except ImportError:
-    try:
-        # Try python-scandir on Python 3.4
-        from scandir import scandir
-    except ImportError:
-        # If python-scandir is not installed, then use a version that is just as slow as listdir.
-        scandir = scandir_generic
-
 
 def scandir_keyfunc(dirent):
     try:
@@ -2469,7 +2408,7 @@ def prepare_subprocess_env(system, env=None):
         # (non-matching) libraries from there.
         # thus we install the original LDLP, before pyinstaller has modified it:
         lp_key = 'LD_LIBRARY_PATH'
-        lp_orig = env.get(lp_key + '_ORIG')  # pyinstaller >= 20160820 / v3.2.1 has this
+        lp_orig = env.get(lp_key + '_ORIG')
         if lp_orig is not None:
             env[lp_key] = lp_orig
         else:

src/borg/testsuite/archiver.py

@@ -61,8 +61,6 @@ from .platform import fakeroot_detected
 from .upgrader import make_attic_repo
 from . import key
 
-# 3.4.3 == first version with argparse bugfix for nargs='*' and 0 arguments given
-argparse_nargs0_fixed = sys.version_info >= (3, 4, 3)
 
 src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 
@@ -1053,10 +1051,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         self.cmd('init', '--encryption=repokey', self.repository_location)
         self.create_regular_file('file1', size=1024 * 80)
         self.create_regular_file('file2', size=1024 * 80)
-        if argparse_nargs0_fixed:
-            output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test')
-            self.assert_in("A input/file1", output)
-            self.assert_in("A input/file2", output)
+        output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test')
+        self.assert_in("A input/file1", output)
+        self.assert_in("A input/file2", output)
 
     def test_create_pattern(self):
         """test file patterns during create"""
@@ -3850,13 +3847,7 @@ class TestCommonOptions:
 
     @pytest.fixture
     def subparsers(self, basic_parser):
-        if sys.version_info >= (3, 7):
-            # py37 pre-release defaults to unwanted required=True, in 3.7.0+ it was fixed to =False
-            return basic_parser.add_subparsers(title='required arguments', metavar='<command>', required=False)
-        else:
-            # py36 does not support required=... argument (but behaves like required=False).
-            # note: use below call for 3.6 and 3.7 when there are no alphas/betas/RCs of 3.7.0 around any more.
-            return basic_parser.add_subparsers(title='required arguments', metavar='<command>')
+        return basic_parser.add_subparsers(title='required arguments', metavar='<command>')
 
     @pytest.fixture
     def parser(self, basic_parser):

tox.ini

@@ -2,7 +2,7 @@
 # fakeroot -u tox --recreate
 
 [tox]
-envlist = py{34,35,36,37,38,39},flake8
+envlist = py{35,36,37,38,39},flake8
 
 [testenv]
 deps =