Merge branch '1.0-maint'

Vagrantfile

@@ -42,7 +42,7 @@ def packages_redhatted
     # needed to compile msgpack-python (otherwise it will use slow fallback code):
     yum install -y gcc-c++
     # for building python:
-    yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz-devel sqlite-devel
+    yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel
     #yum install -y python-pip
     #pip install virtualenv
     touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
@@ -53,10 +53,10 @@ def packages_darwin
   return <<-EOF
     # install all the (security and other) updates
     sudo softwareupdate --install --all
-    # get osxfuse 3.0.x pre-release code from github:
-    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.2.0/osxfuse-3.2.0.dmg >osxfuse.dmg
+    # get osxfuse 3.x pre-release code from github:
+    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.3.3/osxfuse-3.3.3.dmg >osxfuse.dmg
     MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
-    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.2.0.pkg" -target /
+    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.3.3.pkg" -target /
     sudo chown -R vagrant /usr/local  # brew must be able to create stuff here
     ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
     brew update
@@ -321,9 +321,7 @@ Vagrant.configure(2) do |config|
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32")
     b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32")
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_32")
-    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_32")
-    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_32")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("centos6_32")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32")
   end
 
@@ -336,9 +334,7 @@ Vagrant.configure(2) do |config|
     b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64")
     b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64")
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_64")
-    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_64")
-    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("centos6_64")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64")
   end
 
@@ -449,7 +445,7 @@ Vagrant.configure(2) do |config|
     end
     b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("netbsd64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("netbsd64")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64")
   end
 end

docs/api.rst

@@ -42,10 +42,6 @@ API Documentation
     :members:
     :undoc-members:
 
-.. automodule:: borg.hash_sizes
-    :members:
-    :undoc-members:
-
 .. automodule:: borg.xattr
     :members:
     :undoc-members:

docs/changes.rst

@@ -122,8 +122,26 @@ Other changes:
   - vagrant: add ubuntu/xenial 64bit - this box has still some issues
   - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945
 
-Version 1.0.4 (not released yet)
---------------------------------
+
+Version 1.0.5 (2016-07-07)
+--------------------------
+
+Bug fixes:
+
+- borg mount: fix FUSE crash in xattr code on Linux introduced in 1.0.4, #1282
+
+Other changes:
+
+- backport some FAQ entries from master branch
+- add release helper scripts
+- Vagrantfile:
+
+  - centos6: no FUSE, don't build binary
+  - add xz for redhat-like dists
+
+
+Version 1.0.4 (2016-07-07)
+--------------------------
 
 New features:
 
@@ -155,6 +173,13 @@ Bug fixes:
 - Add missing error handler in directory attr restore loop.
 - repo: make sure write data hits disk before the commit tag (#1236) and also
   sync the containing directory.
+- FUSE: getxattr fail must use errno.ENOATTR, #1126
+  (fixes Mac OS X Finder malfunction: "zero bytes" file length, access denied)
+- borg check --repair: do not lose information about the good/original chunks.
+  If we do not lose the original chunk IDs list when "repairing" a file
+  (replacing missing chunks with all-zero chunks), we have a chance to "heal"
+  the file back into its original state later, in case the chunks re-appear
+  (e.g. in a fresh backup). Healing is not implemented yet, see #148.
 - fixes for --read-special mode:
 
   - ignore known files cache, #1241
@@ -204,6 +229,7 @@ Other changes:
   - add note that Fnmatch is default, #1247
   - make clear that lzma levels > 6 are a waste of cpu cycles
   - add a "do not edit" note to auto-generated files, #1250
+  - update cygwin installation docs
 - repository interoperability with borg master (1.1dev) branch:
 
   - borg check: read item metadata keys from manifest, #1147
@@ -211,8 +237,12 @@ Other changes:
   - fix hints file "unknown version" error handling bug
 - tests: add tests for format_line
 - llfuse: update version requirement for freebsd
-- Vagrantfile: use openbsd 5.9, #716
-- use Python 3.5.2 to build the binaries
+- Vagrantfile:
+
+  - use openbsd 5.9, #716
+  - do not install llfuse on netbsd (broken)
+  - update OSXfuse to version 3.3.3
+  - use Python 3.5.2 to build the binaries
 - glibc compatibility checker: scripts/glibc_check.py
 - add .eggs to .gitignore
 

docs/faq.rst

@@ -142,7 +142,7 @@ C to delete all backups residing on S.
 
 These are your options to protect against that:
 
-- Do not allow to permanently delete data from the repo, see :ref:`append-only-mode`.
+- Do not allow to permanently delete data from the repo, see :ref:`append_only_mode`.
 - Use a pull-mode setup using ``ssh -R``, see :issue:`900`.
 - Mount C's filesystem on another machine and then create a backup of it.
 - Do not give C filesystem-level access to S.

docs/installation.rst

@@ -217,15 +217,14 @@ Cygwin
 
 .. note::
     Running under Cygwin is experimental and has only been tested with Cygwin
-    (x86-64) v2.1.0.
+    (x86-64) v2.5.2.
 
 Use the Cygwin installer to install the dependencies::
 
     python3 python3-setuptools
-    python3-cython  # not needed for releases
     binutils gcc-g++
     libopenssl openssl-devel
-    liblz4_1 liblz4-devel  # from cygwinports.org
+    liblz4_1 liblz4-devel
     git make openssh
 
 You can then install ``pip`` and ``virtualenv``::
@@ -233,10 +232,6 @@ You can then install ``pip`` and ``virtualenv``::
     easy_install-3.4 pip
     pip install virtualenv
 
-In case the creation of the virtual environment fails, try deleting this file::
-
-    /usr/lib/python3.4/__pycache__/platform.cpython-34.pyc
-
 
 .. _pip-installation:
 

docs/usage.rst

@@ -805,7 +805,7 @@ Now, let's see how to restore some LVs from such a backup. ::
     $ borg extract --stdout /path/to/repo::arch dev/vg0/home-snapshot > /dev/vg0/home
 
 
-.. _append-only-mode:
+.. _append_only_mode:
 
 Append-only mode
 ~~~~~~~~~~~~~~~~

docs/usage/help.rst.inc

@@ -1,5 +1,43 @@
 .. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
 
+.. _borg_placeholders:
+
+borg help placeholders
+~~~~~~~~~~~~~~~~~~~~~~
+::
+
+
+Repository (or Archive) URLs and --prefix values support these placeholders:
+
+{hostname}
+
+    The (short) hostname of the machine.
+
+{fqdn}
+
+    The full name of the machine.
+
+{now}
+
+    The current local date and time.
+
+{utcnow}
+
+    The current UTC date and time.
+
+{user}
+
+    The user name (or UID, if no name is available) of the user running borg.
+
+{pid}
+
+    The current process ID.
+
+Examples::
+
+    borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
+    borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
+    borg prune --prefix '{hostname}-' ...
 .. _borg_patterns:
 
 borg help patterns
@@ -93,41 +131,3 @@ Examples::
     sh:/home/*/.thumbnails
     EOF
     $ borg create --exclude-from exclude.txt backup /
-.. _borg_placeholders:
-
-borg help placeholders
-~~~~~~~~~~~~~~~~~~~~~~
-::
-
-
-Repository (or Archive) URLs and --prefix values support these placeholders:
-
-{hostname}
-
-    The (short) hostname of the machine.
-
-{fqdn}
-
-    The full name of the machine.
-
-{now}
-
-    The current local date and time.
-
-{utcnow}
-
-    The current UTC date and time.
-
-{user}
-
-    The user name (or UID, if no name is available) of the user running borg.
-
-{pid}
-
-    The current process ID.
-
-Examples::
-
-    borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
-    borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
-    borg prune --prefix '{hostname}-' ...

scripts/release

@@ -0,0 +1,2 @@
+python setup.py register sdist upload --identity="Thomas Waldmann" --sign
+

scripts/sign-binaries

@@ -0,0 +1,9 @@
+#!/bin/bash
+# usage: sign-binaries 201512312359
+
+for file in dist/borg-*; do
+    gpg --armor --detach-sign $file
+done
+
+touch -t $1 dist/*
+

src/borg/archive.py

@@ -1041,11 +1041,12 @@ class ArchiveChecker:
             """
             offset = 0
             chunk_list = []
+            chunks_replaced = False
             for chunk_id, size, csize in item.chunks:
                 if chunk_id not in self.chunks:
                     # If a file chunk is missing, create an all empty replacement chunk
                     logger.error('{}: Missing file chunk detected (Byte {}-{})'.format(item.path, offset, offset + size))
-                    self.error_found = True
+                    self.error_found = chunks_replaced = True
                     data = bytes(size)
                     chunk_id = self.key.id_hash(data)
                     cdata = self.key.encrypt(Chunk(data))
@@ -1055,6 +1056,9 @@ class ArchiveChecker:
                     add_reference(chunk_id, size, csize)
                 chunk_list.append((chunk_id, size, csize))
                 offset += size
+            if chunks_replaced and 'chunks_healthy' not in item:
+                # if this is first repair, remember the correct chunk IDs, so we can maybe heal the file later
+                item.chunks_healthy = item.chunks
             item.chunks = chunk_list
 
         def robust_iterator(archive):

src/borg/constants.py

@@ -1,5 +1,5 @@
 # this set must be kept complete, otherwise the RobustUnpacker might malfunction:
-ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'hardlink_master',
+ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master',
                        'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime',
                        'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended', ])
 

src/borg/fuse.py

@@ -193,7 +193,7 @@ class FuseOperations(llfuse.Operations):
         try:
             return item.get('xattrs', {})[name]
         except KeyError:
-            raise llfuse.FUSEError(errno.ENODATA) from None
+            raise llfuse.FUSEError(llfuse.ENOATTR) from None
 
     def _load_pending_archive(self, inode):
         # Check if this is an archive we need to load

src/borg/item.py

@@ -148,6 +148,7 @@ class Item(PropDict):
     hardlink_master = PropDict._make_property('hardlink_master', bool)
 
     chunks = PropDict._make_property('chunks', (list, type(None)), 'list or None')
+    chunks_healthy = PropDict._make_property('chunks_healthy', (list, type(None)), 'list or None')
 
     xattrs = PropDict._make_property('xattrs', StableDict)