Merge branch '1.0-maint' into merge/1.0-maint

AUTHORS

@@ -1,5 +1,5 @@
-Borg Contributors ("The Borg Collective")
-=========================================
+Borg authors ("The Borg Collective")
+------------------------------------
 
 - Thomas Waldmann <tw@waldmann-edv.de>
 - Antoine Beaupré <anarcat@debian.org>

README.rst

@@ -3,7 +3,7 @@
 .. highlight:: bash
 
 What is BorgBackup?
-===================
+-------------------
 
 BorgBackup (short: Borg) is a deduplicating backup program.
 Optionally, it supports compression and authenticated encryption.
@@ -20,7 +20,7 @@ downloaded Borg, ``docs/installation.rst`` to get started with Borg.
 .. _installation manual: https://borgbackup.readthedocs.org/en/stable/installation.html
 
 Main features
--------------
+~~~~~~~~~~~~~
 
 **Space efficient storage**
   Deduplication based on content-defined chunking is used to reduce the number
@@ -78,11 +78,11 @@ Main features
 
 **Free and Open Source Software**
   * security and functionality can be audited independently
-  * licensed under the BSD (3-clause) license
-
+  * licensed under the BSD (3-clause) license, see `License`_ for the
+    complete license
 
 Easy to use
------------
+~~~~~~~~~~~
 
 Initialize a new backup repository and create a backup archive::
 
@@ -113,24 +113,8 @@ Now doing another backup, just to show off the great deduplication:
 
 For a graphical frontend refer to our complementary project `BorgWeb <https://borgweb.readthedocs.io/>`_.
 
-Checking Release Authenticity and Security Contact
-==================================================
-
-`Releases <https://github.com/borgbackup/borg/releases>`_ are signed with this GPG key,
-please use GPG to verify their authenticity.
-
-In case you discover a security issue, please use this contact for reporting it privately
-and please, if possible, use encrypted E-Mail:
-
-Thomas Waldmann <tw@waldmann-edv.de>
-
-GPG Key Fingerprint: 6D5B EF9A DD20 7580 5747  B70F 9F88 FB52 FAF7 B393
-
-The public key can be fetched from any GPG keyserver, but be careful: you must
-use the **full fingerprint** to check that you got the correct key.
-
 Links
-=====
+-----
 
 * `Main Web Site <https://borgbackup.readthedocs.org/>`_
 * `Releases <https://github.com/borgbackup/borg/releases>`_,
@@ -142,38 +126,10 @@ Links
 * `Web-Chat (IRC) <http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8>`_ and
   `Mailing List <https://mail.python.org/mailman/listinfo/borgbackup>`_
 * `License <https://borgbackup.readthedocs.org/en/stable/authors.html#license>`_
+* `Security contact <https://borgbackup.readthedocs.org/en/stable/support.html#security-contact>`_
 
-Notes
------
-
-Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_".
-
-.. _Attic: https://github.com/jborg/attic
-.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html
-
-Differences between Attic and Borg
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Here's a (incomplete) list of some major changes:
-
-* more open, faster paced development (see `issue #1 <https://github.com/borgbackup/borg/issues/1>`_)
-* lots of attic issues fixed (see `issue #5 <https://github.com/borgbackup/borg/issues/5>`_)
-* less chunk management overhead (less memory and disk usage for chunks index)
-* faster remote cache resync (useful when backing up multiple machines into same repo)
-* compression: no, lz4, zlib or lzma compression, adjustable compression levels
-* repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode)
-* simple sparse file support, great for virtual machine disk files
-* can read special files (e.g. block devices) or from stdin, write to stdout
-* mkdir-based locking is more compatible than attic's posix locking
-* uses fadvise to not spoil / blow up the fs cache
-* better error messages / exception handling
-* better logging, screen output, progress indication
-* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X
-
-Please read the `ChangeLog`_ (or ``docs/changes.rst`` in the source distribution) for more
-information.
-
-BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC (but there is a one-way conversion).
+Compatibility notes
+-------------------
 
 EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
 CHANGES (like when going from 0.x.y to 1.0.0 or from 1.x.y to 2.0.0).
@@ -182,7 +138,8 @@ NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
 
 THIS IS SOFTWARE IN DEVELOPMENT, DECIDE YOURSELF WHETHER IT FITS YOUR NEEDS.
 
-Borg is distributed under a 3-clause BSD license, see `License`_ for the complete license.
+Security issues should be reported to the `Security contact`_ (or
+see ``docs/suppport.rst`` in the source distribution).
 
 |doc| |build| |coverage| |bestpractices|
 

Vagrantfile

@@ -123,8 +123,6 @@ def packages_openbsd
     easy_install-3.4 pip
     pip3 install virtualenv
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
-    # avoid that breaking llfuse install breaks borgbackup install under tox:
-    sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
   EOF
 end
 
@@ -150,11 +148,60 @@ def packages_netbsd
     easy_install-3.4 pip
     pip install virtualenv
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
-    # fuse does not work good enough (see above), do not install llfuse:
-    sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini
   EOF
 end
 
+# Install required cygwin packages and configure environment
+#
+# Microsoft/EdgeOnWindows10 image has MLS-OpenSSH installed by default,
+# which is based on cygwin x86_64 but should not be used together with cygwin.
+# In order to have have cygwin compatible bash 'ImagePath' is replaced with
+# cygrunsrv of newly installed cygwin
+#
+# supported cygwin versions:
+#   x86_64
+#   x86
+def packages_cygwin(version)
+  setup_exe = "setup-#{version}.exe"
+
+  return <<-EOF
+    mkdir -p /cygdrive/c/cygwin
+    powershell -Command '$client = new-object System.Net.WebClient; $client.DownloadFile("https://www.cygwin.com/#{setup_exe}","C:\\cygwin\\#{setup_exe}")'
+    echo '
+    REM --- Change to use different CygWin platform and final install path
+    set CYGSETUP=#{setup_exe}
+    REM --- Install build version of CygWin in a subfolder
+    set OURPATH=%cd%
+	    set CYGBUILD="C:\\cygwin\\CygWin"
+	    set CYGMIRROR=ftp://mirrors.kernel.org/sourceware/cygwin/
+	    set BUILDPKGS=python3,python3-setuptools,binutils,gcc-g++,libopenssl,openssl-devel,git,make,openssh,liblz4-devel,liblz4_1,rsync,curl,python-devel
+    %CYGSETUP% -q -B -o -n -R %CYGBUILD% -L -D -s %CYGMIRROR% -P %BUILDPKGS%
+    cd /d C:\\cygwin\\CygWin\\bin
+    regtool set /HKLM/SYSTEM/CurrentControlSet/Services/OpenSSHd/ImagePath "C:\\cygwin\\CygWin\\bin\\cygrunsrv.exe"
+    bash -c "ssh-host-config --no"
+	    ' > /cygdrive/c/cygwin/install.bat
+    cd /cygdrive/c/cygwin && cmd.exe /c install.bat
+
+    echo "alias mkdir='mkdir -p'" > ~/.profile
+    echo "export CYGWIN_ROOT=/cygdrive/c/cygwin/CygWin" >> ~/.profile
+    echo 'export PATH=$PATH:$CYGWIN_ROOT/bin' >> ~/.profile
+
+    echo '' > ~/.bash_profile
+
+    cmd.exe /c 'setx /m PATH "%PATH%;C:\\cygwin\\CygWin\\bin"'
+    source ~/.profile
+    echo 'db_home: windows' > $CYGWIN_ROOT/etc/nsswitch.conf
+  EOF
+end
+
+def install_cygwin_venv
+  return <<-EOF
+      easy_install-3.4 pip
+      pip install virtualenv
+  EOF
+end
+
+
 def install_pyenv(boxname)
   return <<-EOF
     curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
@@ -231,6 +278,8 @@ def install_borg_no_fuse(boxname)
     rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
     pip install -r requirements.d/development.txt
     pip install -e .
+    # do not install llfuse into the virtualenvs built by tox:
+    sed -i.bak '/fuse.txt/d' tox.ini
   EOF
 end
 
@@ -469,4 +518,31 @@ Vagrant.configure(2) do |config|
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("netbsd64")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64")
   end
+
+  config.vm.define "windows10" do |b|
+    b.vm.box = "Microsoft/EdgeOnWindows10"
+    b.vm.guest = :windows
+    b.vm.boot_timeout = 180
+    b.vm.graceful_halt_timeout = 120
+
+    b.ssh.shell = "sh -l"
+    b.ssh.username = "IEUser"
+    b.ssh.password = "Passw0rd!"
+    b.ssh.insert_key = false
+
+    b.vm.provider :virtualbox do |v|
+      v.memory = 2048
+      #v.gui = true
+    end
+
+    # fix permissions placeholder
+    b.vm.provision "fix perms", :type => :shell,  :privileged => false, :inline => "echo 'fix permission placeholder'"
+
+    b.vm.provision "packages cygwin", :type => :shell, :privileged => false, :inline => packages_cygwin("x86_64")
+    b.vm.provision :reload
+    b.vm.provision "cygwin install pip", :type => :shell, :privileged => false, :inline => install_cygwin_venv
+    b.vm.provision "cygwin build env", :type => :shell, :privileged => false, :inline => build_sys_venv("windows10")    
+    b.vm.provision "cygwin install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("windows10")
+    b.vm.provision "cygwin run tests", :type => :shell, :privileged => false, :inline => run_tests("windows10")
+  end
 end

docs/authors.rst

@@ -1,5 +1,8 @@
 .. include:: global.rst.inc
 
+Authors
+=======
+
 .. include:: ../AUTHORS
 
 License

docs/book.rst

@@ -0,0 +1,22 @@
+.. include:: global.rst.inc
+
+Borg documentation
+==================
+
+.. when you add an element here, do not forget to add it to index.rst
+
+.. toctree::
+    :maxdepth: 2
+
+    introduction
+    installation
+    quickstart
+    usage
+    deployment
+    faq
+    support
+    resources
+    changes
+    internals
+    development
+    authors

docs/conf.py

@@ -193,18 +193,18 @@ htmlhelp_basename = 'borgdoc'
 #latex_paper_size = 'letter'
 
 # The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
+#latex_font_size = '12pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'Borg.tex', 'Borg Documentation',
-   'see "AUTHORS" file', 'manual'),
+  ('book', 'Borg.tex', 'Borg Documentation',
+   'The Borg Collective', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+latex_logo = '_static/logo.png'
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
@@ -214,7 +214,7 @@ latex_documents = [
 #latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+latex_show_urls = 'footnote'
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''

docs/faq.rst

@@ -203,6 +203,13 @@ Thus:
 - have media at another place
 - have a relatively recent backup on your media
 
+How do I report security issue with |project_name|?
+---------------------------------------------------
+
+Send a private email to the :ref:`security-contact` if you think you
+have discovered a security issue. Please disclose security issues
+responsibly.
+
 Why do I get "connection closed by remote" after a while?
 ---------------------------------------------------------
 
@@ -508,3 +515,32 @@ Borg intends to be:
   * do not break compatibility accidentally, without a good reason
     or without warning. allow compatibility breaking for other cases.
   * if major version number changes, it may have incompatible changes
+
+What are the differences between Attic and Borg?
+------------------------------------------------
+
+Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_".
+
+.. _Attic: https://github.com/jborg/attic
+.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html
+
+Here's a (incomplete) list of some major changes:
+
+* more open, faster paced development (see `issue #1 <https://github.com/borgbackup/borg/issues/1>`_)
+* lots of attic issues fixed (see `issue #5 <https://github.com/borgbackup/borg/issues/5>`_)
+* less chunk management overhead (less memory and disk usage for chunks index)
+* faster remote cache resync (useful when backing up multiple machines into same repo)
+* compression: no, lz4, zlib or lzma compression, adjustable compression levels
+* repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode)
+* simple sparse file support, great for virtual machine disk files
+* can read special files (e.g. block devices) or from stdin, write to stdout
+* mkdir-based locking is more compatible than attic's posix locking
+* uses fadvise to not spoil / blow up the fs cache
+* better error messages / exception handling
+* better logging, screen output, progress indication
+* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X
+
+Please read the `ChangeLog`_ (or ``docs/changes.rst`` in the source distribution) for more
+information.
+
+Borg is not compatible with original attic (but there is a one-way conversion).

docs/index.rst

@@ -6,6 +6,8 @@ Borg Documentation
 
 .. include:: ../README.rst
 
+.. when you add an element here, do not forget to add it to book.rst
+
 .. toctree::
    :maxdepth: 2
 

docs/installation.rst

@@ -88,6 +88,9 @@ github to followup on packaging efforts.
 Standalone Binary
 -----------------
 
+.. note:: Releases are signed with an OpenPGP key, see
+          :ref:`security-contact` for more instructions.
+
 |project_name| binaries (generated with `pyinstaller`_) are available
 on the releases_ page for the following platforms:
 

docs/introduction.rst

@@ -0,0 +1,8 @@
+Introduction
+============
+
+.. this shim is here to fix the structure in the PDF
+   rendering. without this stub, the elements in the toctree of
+   index.rst show up a level below the README file included
+
+.. include:: ../README.rst

docs/support.rst

@@ -56,3 +56,21 @@ As a developer, you can become a Bounty Hunter and win bounties (earn money) by
 contributing to |project_name|, a free and open source software project.
 
 We might also use BountySource to fund raise for some bigger goals.
+
+.. _security-contact:
+
+Security
+--------
+
+In case you discover a security issue, please use this contact for reporting it privately
+and please, if possible, use encrypted E-Mail:
+
+Thomas Waldmann <tw@waldmann-edv.de>
+
+GPG Key Fingerprint: 6D5B EF9A DD20 7580 5747  B70F 9F88 FB52 FAF7 B393
+
+The public key can be fetched from any GPG keyserver, but be careful: you must
+use the **full fingerprint** to check that you got the correct key.
+
+`Releases <https://github.com/borgbackup/borg/releases>`_ are signed with this GPG key,
+please use GPG to verify their authenticity.

src/borg/testsuite/archiver.py

@@ -223,6 +223,7 @@ class ArchiverTestCaseBase(BaseTestCase):
         os.environ['BORG_KEYS_DIR'] = self.keys_path
         os.environ['BORG_CACHE_DIR'] = self.cache_path
         os.mkdir(self.input_path)
+        os.chmod(self.input_path, 0o777)  # avoid troubles with fakeroot / FUSE
         os.mkdir(self.output_path)
         os.mkdir(self.keys_path)
         os.mkdir(self.cache_path)
@@ -2154,6 +2155,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         self._test_verify_data('--encryption', 'none')
 
 
+@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteArchiverTestCase(ArchiverTestCase):
     prefix = '__testsuite__:'
 

src/borg/testsuite/repository.py

@@ -632,6 +632,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
             self.assert_equal(self.repository.get(H(0)), b'data2')
 
 
+@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteRepositoryTestCase(RepositoryTestCase):
 
     def open(self, create=False):
@@ -662,6 +663,7 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
         assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--umask=077', '--info']
 
 
+@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
 
     def open(self, create=False):