Merge tag '0.27.0' into multithreading

tagged/signed 0.27.0

.coveragerc

@@ -5,6 +5,7 @@ omit =
     borg/__init__.py
     borg/__main__.py
     borg/_version.py
+    borg/support/*.py
 
 [report]
 exclude_lines =

.gitignore

@@ -23,3 +23,4 @@ borg.build/
 borg.dist/
 borg.exe
 .coverage
+.vagrant

CHANGES.rst

@@ -1,14 +1,66 @@
 Borg Changelog
 ==============
 
+Version 0.27.0
+--------------
+
+New features:
+
+- "borg upgrade" command - attic -> borg one time converter / migration, #21
+- temporary hack to avoid using lots of disk space for chunks.archive.d, #235:
+  To use it: rm -rf chunks.archive.d ; touch chunks.archive.d
+- respect XDG_CACHE_HOME, attic #181
+- add support for arbitrary SSH commands, attic #99
+- borg delete --cache-only REPO (only delete cache, not REPO), attic #123
+
+
+Bug fixes:
+
+- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow
+  down observed when running the Centos6-built binary on Ubuntu, #222
+- do not crash on empty lock.roster, fixes #232
+- fix multiple issues with the cache config version check, #234
+- fix segment entry header size check, attic #352
+  plus other error handling improvements / code deduplication there.
+- always give segment and offset in repo IntegrityErrors
+
+
+Other changes:
+
+- stop producing binary wheels, remove docs about it, #147
+- docs:
+  - add warning about prune
+  - generate usage include files only as needed
+  - development docs: add Vagrant section
+  - update / improve / reformat FAQ
+  - hint to single-file pyinstaller binaries from README
+
+
+Version 0.26.1
+--------------
 
-Version 0.26.0 (not released yet)
----------------------------------
+This is a minor update, just docs and new pyinstaller binaries.
+
+- docs update about python and binary requirements
+- better docs for --read-special, fix #220
+- re-built the binaries, fix #218 and #213 (glibc version issue)
+- update web site about single-file pyinstaller binaries
+
+Note: if you did a python-based installation, there is no need to upgrade.
+
+
+Version 0.26.0
+--------------
 
 New features:
 
+- Faster cache sync (do all in one pass, remove tar/compression stuff), #163
 - BORG_REPO env var to specify the default repo, #168
 - read special files as if they were regular files, #79
+- implement borg create --dry-run, attic issue #267
+- Normalize paths before pattern matching on OS X, #143
+- support OpenBSD and NetBSD (except xattrs/ACLs)
+- support / run tests on Python 3.5
 
 Bug fixes:
 
@@ -16,11 +68,46 @@ Bug fixes:
 - chunker: use off_t to get 64bit on 32bit platform, #178
 - initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0)
 - fix reaction to "no" answer at delete repo prompt, #182
+- setup.py: detect lz4.h header file location
+- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194)
+- fix for obtaining 'char *' from temporary Python value (old code causes
+  a compile error on Mint 17.2)
+- llfuse 0.41 install troubles on some platforms, require < 0.41
+  (UnicodeDecodeError exception due to non-ascii llfuse setup.py)
+- cython code: add some int types to get rid of unspecific python add /
+  subtract operations (avoid undefined symbol FPE_... error on some platforms)
+- fix verbose mode display of stdin backup
+- extract: warn if a include pattern never matched, fixes #209,
+  implement counters for Include/ExcludePatterns
+- archive names with slashes are invalid, attic issue #180
+- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined -
+  fixes building on OpenBSD.
 
 Other changes:
 
 - detect inconsistency / corruption / hash collision, #170
 - replace versioneer with setuptools_scm, #106
+- docs:
+
+  - pkg-config is needed for llfuse installation
+  - be more clear about pruning, attic issue #132
+- unit tests:
+
+  - xattr: ignore security.selinux attribute showing up
+  - ext3 seems to need a bit more space for a sparse file
+  - do not test lzma level 9 compression (avoid MemoryError)
+  - work around strange mtime granularity issue on netbsd, fixes #204
+  - ignore st_rdev if file is not a block/char device, fixes #203
+  - stay away from the setgid and sticky mode bits
+- use Vagrant to do easy cross-platform testing (#196), currently:
+
+  - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit
+  - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit
+  - Centos 7 64bit
+  - FreeBSD 10.2 64bit
+  - OpenBSD 5.7 64bit
+  - NetBSD 6.1.5 64bit
+  - Darwin (OS X Yosemite)
 
 
 Version 0.25.0

MANIFEST.in

@@ -1,7 +1,9 @@
-include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in versioneer.py
+include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in
 recursive-include borg *.pyx
 recursive-include docs *
 recursive-exclude docs *.pyc
 recursive-exclude docs *.pyo
 prune docs/_build
+prune .travis
+exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile
 include borg/_version.py

README.rst

@@ -63,10 +63,15 @@ Main features
     Backup archives are mountable as userspace filesystems for easy interactive
     backup examination and restores (e.g. by using a regular file manager).
 
+**Easy installation**
+    For Linux, Mac OS X and FreeBSD, we offer a single-file pyinstaller binary
+    that does not require installing anything - you can just run it.
+
 **Platforms Borg works on**
   * Linux
-  * FreeBSD
   * Mac OS X
+  * FreeBSD
+  * OpenBSD and NetBSD (for both: no xattrs/ACLs support yet)
   * Cygwin (unsupported)
 
 **Free and Open Source Software**

Vagrantfile

@@ -0,0 +1,427 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Automated creation of testing environments / binaries on misc. platforms
+
+def packages_prepare_wheezy
+  return <<-EOF
+      # debian 7 wheezy does not have lz4, but it is available from wheezy-backports:
+      echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list
+  EOF
+end
+
+def packages_prepare_precise
+  return <<-EOF
+      # ubuntu 12.04 precise does not have lz4, but it is available from a ppa:
+      add-apt-repository -y ppa:gezakovacs/lz4
+  EOF
+end
+
+def packages_debianoid
+  return <<-EOF
+    apt-get update
+    # for building borgbackup and dependencies:
+    apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config
+    apt-get install -y fakeroot build-essential git
+    apt-get install -y python3-dev python3-setuptools
+    # for building python:
+    apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev
+    # this way it works on older dists (like ubuntu 12.04) also:
+    easy_install3 pip
+    pip3 install virtualenv
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def packages_redhatted
+  return <<-EOF
+    yum install -y epel-release
+    yum update -y
+    # for building borgbackup and dependencies:
+    yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig
+    usermod -a -G fuse vagrant
+    yum install -y fakeroot gcc git patch
+    # for building python:
+    yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz-devel sqlite-devel
+    #yum install -y python-pip
+    #pip install virtualenv
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def packages_darwin
+  return <<-EOF
+    # get osxfuse 3.0.x pre-release code from github:
+    curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.0.5/osxfuse-3.0.5.dmg >osxfuse.dmg
+    MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \
+    && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.0.5.pkg" -target /
+    sudo chown -R vagrant /usr/local  # brew must be able to create stuff here
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+    brew update
+    brew install openssl
+    brew install lz4
+    brew install fakeroot
+    brew install git
+    brew install pkgconfig
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def packages_freebsd
+  return <<-EOF
+    # for building borgbackup and dependencies:
+    pkg install -y openssl liblz4 fusefs-libs pkgconf
+    pkg install -y fakeroot git bash
+    # for building python:
+    pkg install sqlite3
+    # make bash default / work:
+    chsh -s bash vagrant
+    mount -t fdescfs fdesc /dev/fd
+    echo 'fdesc	/dev/fd		fdescfs		rw	0	0' >> /etc/fstab
+    # make FUSE work
+    echo 'fuse_load="YES"' >> /boot/loader.conf
+    echo 'vfs.usermount=1' >> /etc/sysctl.conf
+    kldload fuse
+    sysctl vfs.usermount=1
+    pw groupmod operator -M vagrant
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def packages_openbsd
+  return <<-EOF
+    . ~/.profile
+    mkdir -p /home/vagrant/borg
+    rsync -aH /vagrant/borg/ /home/vagrant/borg/
+    rm -rf /vagrant/borg
+    ln -sf /home/vagrant/borg /vagrant/
+    pkg_add bash
+    chsh -s /usr/local/bin/bash vagrant
+    pkg_add openssl
+    pkg_add lz4
+    # pkg_add fuse  # does not install, sdl dependency missing
+    pkg_add git  # no fakeroot
+    pkg_add python-3.4.2
+    pkg_add py3-setuptools
+    ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
+    ln -sf /usr/local/bin/python3.4 /usr/local/bin/python
+    easy_install-3.4 pip
+    pip3 install virtualenv
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def packages_netbsd
+  return <<-EOF
+    hostname netbsd  # the box we use has an invalid hostname
+    PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/"
+    export PKG_PATH
+    pkg_add mozilla-rootcerts lz4 git bash
+    chsh -s bash vagrant
+    mkdir -p /usr/local/opt/lz4/include
+    mkdir -p /usr/local/opt/lz4/lib
+    ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/
+    ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/
+    touch /etc/openssl/openssl.cnf  # avoids a flood of "can't open ..."
+    mozilla-rootcerts install
+    # llfuse does not support netbsd
+    pkg_add python34 py34-setuptools
+    ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python
+    ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3
+    easy_install-3.4 pip
+    pip install virtualenv
+    touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile
+  EOF
+end
+
+def install_pyenv(boxname)
+  return <<-EOF
+    curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
+    echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile
+    echo 'eval "$(pyenv init -)"' >> ~/.bash_profile
+    echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile
+    echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile
+  EOF
+end
+
+def fix_pyenv_darwin(boxname)
+  return <<-EOF
+    echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile
+  EOF
+end
+
+def install_pythons(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    pyenv install 3.2.2  # tests, 3.2(.0) and 3.2.1 deadlock, issue #221
+    pyenv install 3.3.0  # tests
+    pyenv install 3.4.0  # tests
+    pyenv install 3.5.0  # tests
+    #pyenv install 3.5.1  # binary build, use latest 3.5.x release
+    pyenv rehash
+  EOF
+end
+
+def build_sys_venv(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    virtualenv --python=python3 borg-env
+  EOF
+end
+
+def build_pyenv_venv(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    # use the latest 3.5 release
+    pyenv global 3.5.0
+    pyenv virtualenv 3.5.0 borg-env
+    ln -s ~/.pyenv/versions/borg-env .
+  EOF
+end
+
+def install_borg(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    . borg-env/bin/activate
+    pip install -U wheel  # upgrade wheel, too old for 3.5
+    cd borg
+    # clean up (wrong/outdated) stuff we likely got via rsync:
+    rm -f borg/*.so borg/*.cpy*
+    rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
+    rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
+    pip install 'llfuse<0.41'  # 0.41 does not install due to UnicodeDecodeError
+    pip install -r requirements.d/development.txt
+    pip install -e .
+  EOF
+end
+
+def install_pyinstaller(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    . borg-env/bin/activate
+    git clone https://github.com/pyinstaller/pyinstaller.git
+    cd pyinstaller
+    git checkout master
+    pip install -e .
+  EOF
+end
+
+def install_pyinstaller_bootloader(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    . borg-env/bin/activate
+    git clone https://github.com/pyinstaller/pyinstaller.git
+    cd pyinstaller
+    git checkout master
+    # build bootloader, if it is not included
+    cd bootloader
+    python ./waf all
+    cd ..
+    pip install -e .
+  EOF
+end
+
+def build_binary_with_pyinstaller(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg
+    . borg-env/bin/activate
+    cd borg
+    pyinstaller -F -n borg --hidden-import=logging.config borg/__main__.py
+  EOF
+end
+
+def run_tests(boxname)
+  return <<-EOF
+    . ~/.bash_profile
+    cd /vagrant/borg/borg
+    . ../borg-env/bin/activate
+    if which pyenv > /dev/null; then
+      # for testing, use the earliest point releases of the supported python versions:
+      pyenv global 3.2.2 3.3.0 3.4.0 3.5.0
+    fi
+    # otherwise: just use the system python
+    if which fakeroot > /dev/null; then
+      fakeroot -u tox --skip-missing-interpreters
+    else
+      tox --skip-missing-interpreters
+    fi
+  EOF
+end
+
+def fix_perms
+  return <<-EOF
+    # . ~/.profile
+    chown -R vagrant /vagrant/borg
+  EOF
+end
+
+Vagrant.configure(2) do |config|
+  # use rsync to copy content to the folder
+  config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync"
+  # do not let the VM access . on the host machine via the default shared folder!
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+
+  # fix permissions on synced folder
+  config.vm.provision "fix perms", :type => :shell, :inline => fix_perms
+
+  config.vm.provider :virtualbox do |v|
+    #v.gui = true
+    v.cpus = 1
+  end
+
+  # Linux
+  config.vm.define "centos7_64" do |b|
+    b.vm.box = "centos/7"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos7_64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64")
+  end
+
+  config.vm.define "centos6_32" do |b|
+    b.vm.box = "centos6-32"
+    b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_32")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_32")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_32")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32")
+  end
+
+  config.vm.define "centos6_64" do |b|
+    b.vm.box = "centos6-64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_64")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_64")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64")
+  end
+
+  config.vm.define "trusty64" do |b|
+    b.vm.box = "ubuntu/trusty64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64")
+  end
+
+  config.vm.define "precise32" do |b|
+    b.vm.box = "ubuntu/precise32"
+    b.vm.provision "packages prepare precise", :type => :shell, :inline => packages_prepare_precise
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("precise32")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("precise32")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("precise32")
+  end
+
+  config.vm.define "jessie64" do |b|
+    b.vm.box = "debian/jessie64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("jessie64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64")
+  end
+
+  config.vm.define "wheezy32" do |b|
+    b.vm.box = "boxcutter/debian79-i386"
+    b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy32")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy32")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32")
+  end
+
+  config.vm.define "wheezy64" do |b|
+    b.vm.box = "boxcutter/debian79"
+    b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy
+    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy64")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy64")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64")
+  end
+
+  # OS X
+  config.vm.define "darwin64" do |b|
+    b.vm.box = "jhcook/yosemite-clitools"
+    b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64")
+    b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("darwin64")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("darwin64")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64")
+  end
+
+  # BSD
+  config.vm.define "freebsd64" do |b|
+    b.vm.box = "geoffgarside/freebsd-10.2"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd
+    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd")
+    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("freebsd")
+    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller_bootloader("freebsd")
+    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd")
+  end
+
+  config.vm.define "openbsd64" do |b|
+    b.vm.box = "bodgit/openbsd-5.7-amd64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
+  end
+
+  config.vm.define "netbsd64" do |b|
+    b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64"
+    b.vm.provider :virtualbox do |v|
+      v.memory = 768
+    end
+    b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64")
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("netbsd64")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64")
+  end
+end

borg/_chunker.c

@@ -156,7 +156,7 @@ chunker_fill(Chunker *c, PyThreadState **tstatep)
             return 0;
         }
         length = c->bytes_read - offset;
-        #if ( _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L )
+        #if ( ( _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L ) && defined(POSIX_FADV_DONTNEED) )
         // We tell the OS that we do not need the data that we just have read any
         // more (that it maybe has in the cache). This avoids that we spoil the
         // complete cache with data that we only read once and (due to cache

borg/archiver.py

@@ -1,4 +1,6 @@
-import argparse
+from .support import argparse  # see support/__init__.py docstring
+                               # DEPRECATED - remove after requiring py 3.4
+
 from binascii import hexlify
 from datetime import datetime
 from operator import attrgetter
@@ -15,11 +17,12 @@ import traceback
 from . import __version__
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
 from .compress import Compressor, COMPR_BUFFER
+from .upgrader import AtticRepositoryUpgrader
 from .repository import Repository
 from .cache import Cache
 from .key import key_creator
 from .helpers import Error, location_validator, format_time, format_file_size, \
-    format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
+    format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
     get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec
@@ -288,6 +291,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
             # processing time, archive order is not as traversal order on "create".
             while dirs:
                 archive.extract_item(dirs.pop(-1))
+        for pattern in (patterns or []):
+            if isinstance(pattern, IncludePattern) and  pattern.match_count == 0:
+                self.print_error("Warning: Include pattern '%s' never matched.", pattern)
         return self.exit_code
 
     def do_rename(self, args):
@@ -317,17 +323,19 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
             if args.stats:
                 stats.print_('Deleted data:', cache)
         else:
-            print("You requested to completely DELETE the repository *including* all archives it contains:")
-            for archive_info in manifest.list_archive_infos(sort_by='ts'):
-                print(format_archive(archive_info))
-            if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
-                print("""Type "YES" if you understand this and want to continue.\n""")
-                if input('Do you want to continue? ') != 'YES':
-                    self.exit_code = 1
-                    return self.exit_code
-            repository.destroy()
+            if not args.cache_only:
+                print("You requested to completely DELETE the repository *including* all archives it contains:")
+                for archive_info in manifest.list_archive_infos(sort_by='ts'):
+                    print(format_archive(archive_info))
+                if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
+                    print("""Type "YES" if you understand this and want to continue.\n""")
+                    if input('Do you want to continue? ') != 'YES':
+                        self.exit_code = 1
+                        return self.exit_code
+                repository.destroy()
+                print("Repository deleted.")
             cache.destroy()
-            print("Repository and corresponding cache were deleted.")
+            print("Cache deleted.")
         return self.exit_code
 
     def do_mount(self, args):
@@ -461,6 +469,24 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
             stats.print_('Deleted data:', cache)
         return self.exit_code
 
+    def do_upgrade(self, args):
+        """upgrade a repository from a previous version"""
+        # XXX: currently only upgrades from Attic repositories, but may
+        # eventually be extended to deal with major upgrades for borg
+        # itself.
+        #
+        # in this case, it should auto-detect the current repository
+        # format and fire up necessary upgrade mechanism. this remains
+        # to be implemented.
+
+        # XXX: should auto-detect if it is an attic repository here
+        repo = AtticRepositoryUpgrader(args.repository.path, create=False)
+        try:
+            repo.upgrade(args.dry_run)
+        except NotImplementedError as e:
+            print("warning: %s" % e)
+        return self.exit_code
+
     helptext = {}
     helptext['patterns'] = '''
         Exclude patterns use a variant of shell pattern syntax, with '*' matching any
@@ -549,10 +575,10 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                    help='verbose output')
         common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
                                    help='do not load/update the file metadata cache used to detect unchanged files')
-        common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=0o077, metavar='M',
-                                   help='set umask to M (local and remote, default: 0o077)')
-        common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH',
-                                   help='set remote path to executable (default: "borg")')
+        common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M',
+                                   help='set umask to M (local and remote, default: %(default)s)')
+        common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH',
+                                   help='set remote path to executable (default: "%(default)s")')
 
         # We can't use argparse for "serve" since we don't want it to show up in "Available commands"
         if args:
@@ -720,7 +746,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                help='do not create a backup archive')
         subparser.add_argument('archive', metavar='ARCHIVE',
                                type=location_validator(archive=True),
-                               help='archive to create')
+                               help='name of archive to create (must be also a valid directory name)')
         subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                                help='paths to archive')
 
@@ -791,6 +817,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         subparser.add_argument('-s', '--stats', dest='stats',
                                action='store_true', default=False,
                                help='print statistics for the deleted archive')
+        subparser.add_argument('-c', '--cache-only', dest='cache_only',
+                               action='store_true', default=False,
+                               help='delete only the local cache for the given repository')
         subparser.add_argument('target', metavar='TARGET', nargs='?', default='',
                                type=location_validator(),
                                help='archive or repository to delete')
@@ -864,6 +893,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         If a prefix is set with -p, then only archives that start with the prefix are
         considered for deletion and only those archives count towards the totals
         specified by the rules.
+        Otherwise, *all* archives in the repository are candidates for deletion!
         """)
         subparser = subparsers.add_parser('prune', parents=[common_parser],
                                           description=self.do_prune.__doc__,
@@ -894,6 +924,53 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
                                type=location_validator(archive=False),
                                help='repository to prune')
 
+        upgrade_epilog = textwrap.dedent("""
+        upgrade an existing Borg repository in place. this currently
+        only support converting an Attic repository, but may
+        eventually be extended to cover major Borg upgrades as well.
+
+        it will change the magic strings in the repository's segments
+        to match the new Borg magic strings. the keyfiles found in
+        $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and
+        copied to $BORG_KEYS_DIR or ~/.borg/keys.
+
+        the cache files are converted, from $ATTIC_CACHE_DIR or
+        ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the
+        cache layout between Borg and Attic changed, so it is possible
+        the first backup after the conversion takes longer than expected
+        due to the cache resync.
+
+        it is recommended you run this on a copy of the Attic
+        repository, in case something goes wrong, for example:
+
+            cp -a attic borg
+            borg upgrade -n borg
+            borg upgrade borg
+
+        upgrade should be able to resume if interrupted, although it
+        will still iterate over all segments. if you want to start
+        from scratch, use `borg delete` over the copied repository to
+        make sure the cache files are also removed:
+
+            borg delete borg
+
+        the conversion can PERMANENTLY DAMAGE YOUR REPOSITORY! Attic
+        will also NOT BE ABLE TO READ THE BORG REPOSITORY ANYMORE, as
+        the magic strings will have changed.
+
+        you have been warned.""")
+        subparser = subparsers.add_parser('upgrade', parents=[common_parser],
+                                          description=self.do_upgrade.__doc__,
+                                          epilog=upgrade_epilog,
+                                          formatter_class=argparse.RawDescriptionHelpFormatter)
+        subparser.set_defaults(func=self.do_upgrade)
+        subparser.add_argument('-n', '--dry-run', dest='dry_run',
+                               default=False, action='store_true',
+                               help='do not change repository')
+        subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False),
+                               help='path to the repository to be upgraded')
+
         subparser = subparsers.add_parser('help', parents=[common_parser],
                                           description='Extra help')
         subparser.add_argument('--epilog-only', dest='epilog_only',

borg/cache.py

@@ -1,4 +1,4 @@
-from configparser import RawConfigParser
+import configparser
 from .remote import cache_if_remote
 import errno
 import msgpack
@@ -93,7 +93,7 @@ class Cache:
         os.makedirs(self.path)
         with open(os.path.join(self.path, 'README'), 'w') as fd:
             fd.write('This is a Borg cache')
-        config = RawConfigParser()
+        config = configparser.RawConfigParser()
         config.add_section('cache')
         config.set('cache', 'version', '1')
         config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii'))
@@ -101,8 +101,7 @@ class Cache:
         with open(os.path.join(self.path, 'config'), 'w') as fd:
             config.write(fd)
         ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
-        with open(os.path.join(self.path, 'chunks.archive'), 'wb') as fd:
-            pass  # empty file
+        os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         with open(os.path.join(self.path, 'files'), 'wb') as fd:
             pass  # empty file
 
@@ -114,10 +113,17 @@ class Cache:
         shutil.rmtree(self.path)
 
     def _do_open(self):
-        self.config = RawConfigParser()
-        self.config.read(os.path.join(self.path, 'config'))
-        if self.config.getint('cache', 'version') != 1:
-            raise Exception('%s Does not look like a Borg cache')
+        self.config = configparser.RawConfigParser()
+        config_path = os.path.join(self.path, 'config')
+        self.config.read(config_path)
+        try:
+            cache_version = self.config.getint('cache', 'version')
+            wanted_version = 1
+            if  cache_version != wanted_version:
+                raise Exception('%s has unexpected cache version %d (wanted: %d).' % (
+                    config_path, cache_version, wanted_version))
+        except configparser.NoSectionError as e:
+            raise Exception('%s does not look like a Borg cache.' % config_path)
         self.id = self.config.get('cache', 'repository')
         self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
         self.timestamp = self.config.get('cache', 'timestamp', fallback=None)
@@ -158,7 +164,6 @@ class Cache:
         os.mkdir(txn_dir)
         shutil.copy(os.path.join(self.path, 'config'), txn_dir)
         shutil.copy(os.path.join(self.path, 'chunks'), txn_dir)
-        shutil.copy(os.path.join(self.path, 'chunks.archive'), txn_dir)
         shutil.copy(os.path.join(self.path, 'files'), txn_dir)
         os.rename(os.path.join(self.path, 'txn.tmp'),
                   os.path.join(self.path, 'txn.active'))
@@ -200,7 +205,6 @@ class Cache:
         if os.path.exists(txn_dir):
             shutil.copy(os.path.join(txn_dir, 'config'), self.path)
             shutil.copy(os.path.join(txn_dir, 'chunks'), self.path)
-            shutil.copy(os.path.join(txn_dir, 'chunks.archive'), self.path)
             shutil.copy(os.path.join(txn_dir, 'files'), self.path)
             os.rename(txn_dir, os.path.join(self.path, 'txn.tmp'))
             if os.path.exists(os.path.join(self.path, 'txn.tmp')):
@@ -211,54 +215,34 @@ class Cache:
     def sync(self):
         """Re-synchronize chunks cache with repository.
 
-        If present, uses a compressed tar archive of known backup archive
-        indices, so it only needs to fetch infos from repo and build a chunk
-        index once per backup archive.
-        If out of sync, the tar gets rebuilt from known + fetched chunk infos,
-        so it has complete and current information about all backup archives.
-        Finally, it builds the master chunks index by merging all indices from
-        the tar.
-
-        Note: compression (esp. xz) is very effective in keeping the tar
-              relatively small compared to the files it contains.
+        Maintains a directory with known backup archive indexes, so it only
+        needs to fetch infos from repo and build a chunk index once per backup
+        archive.
+        If out of sync, missing archive indexes get added, outdated indexes
+        get removed and a new master chunks index is built by merging all
+        archive indexes.
         """
-        in_archive_path = os.path.join(self.path, 'chunks.archive')
-        out_archive_path = os.path.join(self.path, 'chunks.archive.tmp')
-
-        def open_in_archive():
-            try:
-                tf = tarfile.open(in_archive_path, 'r')
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-                # file not found
-                tf = None
-            except tarfile.ReadError:
-                # empty file?
-                tf = None
-            return tf
-
-        def open_out_archive():
-            for compression in ('xz', 'bz2', 'gz'):
-                # xz needs py 3.3, bz2 and gz also work on 3.2
-                try:
-                    tf = tarfile.open(out_archive_path, 'w:'+compression, format=tarfile.PAX_FORMAT)
-                    break
-                except tarfile.CompressionError:
-                    continue
-            else:  # shouldn't happen
-                tf = None
-            return tf
-
-        def close_archive(tf):
-            if tf:
-                tf.close()
+        archive_path = os.path.join(self.path, 'chunks.archive.d')
+
+        def mkpath(id, suffix=''):
+            id_hex = hexlify(id).decode('ascii')
+            path = os.path.join(archive_path, id_hex + suffix)
+            return path.encode('utf-8')
+
+        def cached_archives():
+            if self.do_cache:
+                fns = os.listdir(archive_path)
+                # filenames with 64 hex digits == 256bit
+                return set(unhexlify(fn) for fn in fns if len(fn) == 64)
+            else:
+                return set()
 
-        def delete_in_archive():
-            os.unlink(in_archive_path)
+        def repo_archives():
+            return set(info[b'id'] for info in self.manifest.archives.values())
 
-        def rename_out_archive():
-            os.rename(out_archive_path, in_archive_path)
+        def cleanup_outdated(ids):
+            for id in ids:
+                os.unlink(mkpath(id))
 
         def add(chunk_idx, id, size, csize, incr=1):
             try:
@@ -267,16 +251,7 @@ class Cache:
             except KeyError:
                 chunk_idx[id] = incr, size, csize
 
-        def transfer_known_idx(archive_id, tf_in, tf_out):
-            archive_id_hex = hexlify(archive_id).decode('ascii')
-            tarinfo = tf_in.getmember(archive_id_hex)
-            archive_name = tarinfo.pax_headers['archive_name']
-            print('Already known archive:', archive_name)
-            f_in = tf_in.extractfile(archive_id_hex)
-            tf_out.addfile(tarinfo, f_in)
-            return archive_name
-
-        def fetch_and_build_idx(archive_id, repository, key, tmp_dir, tf_out):
+        def fetch_and_build_idx(archive_id, repository, key):
             chunk_idx = ChunkIndex()
             cdata = repository.get(archive_id)
             data = key.decrypt(archive_id, cdata)
@@ -285,7 +260,6 @@ class Cache:
             if archive[b'version'] != 1:
                 raise Exception('Unknown archive metadata version')
             decode_dict(archive, (b'name',))
-            print('Analyzing new archive:', archive[b'name'])
             unpacker = msgpack.Unpacker()
             for item_id, chunk in zip(archive[b'items'], repository.get_many(archive[b'items'])):
                 data = key.decrypt(item_id, chunk)
@@ -298,56 +272,76 @@ class Cache:
                     if b'chunks' in item:
                         for chunk_id, size, csize in item[b'chunks']:
                             add(chunk_idx, chunk_id, size, csize)
-            archive_id_hex = hexlify(archive_id).decode('ascii')
-            file_tmp = os.path.join(tmp_dir, archive_id_hex).encode('utf-8')
-            chunk_idx.write(file_tmp)
-            tarinfo = tf_out.gettarinfo(file_tmp, archive_id_hex)
-            tarinfo.pax_headers['archive_name'] = archive[b'name']
-            with open(file_tmp, 'rb') as f:
-                tf_out.addfile(tarinfo, f)
-            os.unlink(file_tmp)
-
-        def create_master_idx(chunk_idx, tf_in, tmp_dir):
+            if self.do_cache:
+                fn = mkpath(archive_id)
+                fn_tmp = mkpath(archive_id, suffix='.tmp')
+                try:
+                    chunk_idx.write(fn_tmp)
+                except Exception:
+                    os.unlink(fn_tmp)
+                else:
+                    os.rename(fn_tmp, fn)
+            return chunk_idx
+
+        def lookup_name(archive_id):
+            for name, info in self.manifest.archives.items():
+                if info[b'id'] == archive_id:
+                    return name
+
+        def create_master_idx(chunk_idx):
+            print('Synchronizing chunks cache...')
+            cached_ids = cached_archives()
+            archive_ids = repo_archives()
+            print('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.' % (
+                len(archive_ids), len(cached_ids),
+                len(cached_ids - archive_ids), len(archive_ids - cached_ids), ))
+            # deallocates old hashindex, creates empty hashindex:
             chunk_idx.clear()
-            for tarinfo in tf_in:
-                archive_id_hex = tarinfo.name
-                archive_name = tarinfo.pax_headers['archive_name']
-                print("- extracting archive %s ..." % archive_name)
-                tf_in.extract(archive_id_hex, tmp_dir)
-                chunk_idx_path = os.path.join(tmp_dir, archive_id_hex).encode('utf-8')
-                print("- reading archive ...")
-                archive_chunk_idx = ChunkIndex.read(chunk_idx_path)
-                print("- merging archive ...")
-                chunk_idx.merge(archive_chunk_idx)
-                os.unlink(chunk_idx_path)
+            cleanup_outdated(cached_ids - archive_ids)
+            if archive_ids:
+                chunk_idx = None
+                for archive_id in archive_ids:
+                    archive_name = lookup_name(archive_id)
+                    if archive_id in cached_ids:
+                        archive_chunk_idx_path = mkpath(archive_id)
+                        print("Reading cached archive chunk index for %s ..." % archive_name)
+                        archive_chunk_idx = ChunkIndex.read(archive_chunk_idx_path)
+                    else:
+                        print('Fetching and building archive index for %s ...' % archive_name)
+                        archive_chunk_idx = fetch_and_build_idx(archive_id, repository, self.key)
+                    print("Merging into master chunks index ...")
+                    if chunk_idx is None:
+                        # we just use the first archive's idx as starting point,
+                        # to avoid growing the hash table from 0 size and also
+                        # to save 1 merge call.
+                        chunk_idx = archive_chunk_idx
+                    else:
+                        chunk_idx.merge(archive_chunk_idx)
+            print('Done.')
+            return chunk_idx
+
+        def legacy_cleanup():
+            """bring old cache dirs into the desired state (cleanup and adapt)"""
+            try:
+                os.unlink(os.path.join(self.path, 'chunks.archive'))
+            except:
+                pass
+            try:
+                os.unlink(os.path.join(self.path, 'chunks.archive.tmp'))
+            except:
+                pass
+            try:
+                os.mkdir(archive_path)
+            except:
+                pass
 
         self.begin_txn()
-        print('Synchronizing chunks cache...')
-        # XXX we have to do stuff on disk due to lacking ChunkIndex api
-        with tempfile.TemporaryDirectory(prefix='borg-tmp') as tmp_dir:
-            repository = cache_if_remote(self.repository)
-            out_archive = open_out_archive()
-            in_archive = open_in_archive()
-            if in_archive:
-                known_ids = set(unhexlify(hexid) for hexid in in_archive.getnames())
-            else:
-                known_ids = set()
-            archive_ids = set(info[b'id'] for info in self.manifest.archives.values())
-            print('Rebuilding archive collection. Known: %d Repo: %d Unknown: %d' % (
-                len(known_ids), len(archive_ids), len(archive_ids - known_ids), ))
-            for archive_id in archive_ids & known_ids:
-                transfer_known_idx(archive_id, in_archive, out_archive)
-            close_archive(in_archive)
-            delete_in_archive()  # free disk space
-            for archive_id in archive_ids - known_ids:
-                fetch_and_build_idx(archive_id, repository, self.key, tmp_dir, out_archive)
-            close_archive(out_archive)
-            rename_out_archive()
-            print('Merging collection into master chunks cache...')
-            in_archive = open_in_archive()
-            create_master_idx(self.chunks, in_archive, tmp_dir)
-            close_archive(in_archive)
-            print('Done.')
+        repository = cache_if_remote(self.repository)
+        legacy_cleanup()
+        # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d -
+        # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk)
+        self.do_cache = os.path.isdir(archive_path)
+        self.chunks = create_master_idx(self.chunks)
 
     def add_chunk(self, id, data, stats):
         if not self.txn_active:

borg/chunker.pyx

@@ -20,7 +20,7 @@ cdef extern from "_chunker.c":
 cdef class Chunker:
     cdef _Chunker *chunker
 
-    def __cinit__(self, seed, chunk_min_exp, chunk_max_exp, hash_mask_bits, hash_window_size):
+    def __cinit__(self, int seed, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int hash_window_size):
         min_size = 1 << chunk_min_exp
         max_size = 1 << chunk_max_exp
         hash_mask = (1 << hash_mask_bits) - 1

borg/crypto.pyx

@@ -52,7 +52,7 @@ bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0]
 long_to_bytes = lambda x: _long.pack(x)
 
 
-def num_aes_blocks(length):
+def num_aes_blocks(int length):
     """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data.
        Note: this is only correct for modes without padding, like AES-CTR.
     """

borg/hashindex.pyx

@@ -37,7 +37,8 @@ cdef class IndexBase:
     def __cinit__(self, capacity=0, path=None, key_size=32):
         self.key_size = key_size
         if path:
-            self.index = hashindex_read(os.fsencode(path))
+            path = os.fsencode(path)
+            self.index = hashindex_read(path)
             if not self.index:
                 raise Exception('hashindex_read failed')
         else:
@@ -54,7 +55,8 @@ cdef class IndexBase:
         return cls(path=path)
 
     def write(self, path):
-        if not hashindex_write(self.index, os.fsencode(path)):
+        path = os.fsencode(path)
+        if not hashindex_write(self.index, path):
             raise Exception('hashindex_write failed')
 
     def clear(self):

borg/helpers.py

@@ -1,6 +1,9 @@
-import argparse
+from .support import argparse  # see support/__init__.py docstring
+                               # DEPRECATED - remove after requiring py 3.4
+
 import binascii
 from collections import namedtuple
+from functools import wraps
 import grp
 import os
 import pwd
@@ -8,6 +11,8 @@ import queue
 import re
 import sys
 import time
+import unicodedata
+
 from datetime import datetime, timezone, timedelta
 from fnmatch import translate
 from operator import attrgetter
@@ -170,8 +175,8 @@ def get_keys_dir():
 
 def get_cache_dir():
     """Determine where to repository keys and cache"""
-    return os.environ.get('BORG_CACHE_DIR',
-                          os.path.join(os.path.expanduser('~'), '.cache', 'borg'))
+    xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache'))
+    return os.environ.get('BORG_CACHE_DIR', os.path.join(xdg_cache, 'borg'))
 
 
 def to_localtime(ts):
@@ -223,6 +228,24 @@ def exclude_path(path, patterns):
 # unify the two cases, we add a path separator to the end of
 # the path before matching.
 
+def normalized(func):
+    """ Decorator for the Pattern match methods, returning a wrapper that
+    normalizes OSX paths to match the normalized pattern on OSX, and 
+    returning the original method on other platforms"""
+    @wraps(func)
+    def normalize_wrapper(self, path):
+        return func(self, unicodedata.normalize("NFD", path))
+
+    if sys.platform in ('darwin',):
+        # HFS+ converts paths to a canonical form, so users shouldn't be
+        # required to enter an exact match
+        return normalize_wrapper
+    else:
+        # Windows and Unix filesystems allow different forms, so users
+        # always have to enter an exact match
+        return func
+
+
 class IncludePattern:
     """Literal files or directories listed on the command line
     for some operations (e.g. extract, but not create).
@@ -230,34 +253,61 @@ class IncludePattern:
     path match as well.  A trailing slash makes no difference.
     """
     def __init__(self, pattern):
+        self.pattern_orig = pattern
+        self.match_count = 0
+
+        if sys.platform in ('darwin',):
+            pattern = unicodedata.normalize("NFD", pattern)
+
         self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep
 
+    @normalized
     def match(self, path):
-        return (path+os.path.sep).startswith(self.pattern)
+        matches = (path+os.path.sep).startswith(self.pattern)
+        if matches:
+            self.match_count += 1
+        return matches
 
     def __repr__(self):
         return '%s(%s)' % (type(self), self.pattern)
 
+    def __str__(self):
+        return self.pattern_orig
+
 
 class ExcludePattern(IncludePattern):
     """Shell glob patterns to exclude.  A trailing slash means to
     exclude the contents of a directory, but not the directory itself.
     """
     def __init__(self, pattern):
+        self.pattern_orig = pattern
+        self.match_count = 0
+
         if pattern.endswith(os.path.sep):
             self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep+'*'+os.path.sep
         else:
             self.pattern = os.path.normpath(pattern)+os.path.sep+'*'
+
+        if sys.platform in ('darwin',):
+            self.pattern = unicodedata.normalize("NFD", self.pattern)
+
         # fnmatch and re.match both cache compiled regular expressions.
         # Nevertheless, this is about 10 times faster.
         self.regex = re.compile(translate(self.pattern))
 
+    @normalized
     def match(self, path):
-        return self.regex.match(path+os.path.sep) is not None
+        matches = self.regex.match(path+os.path.sep) is not None
+        if matches:
+            self.match_count += 1
+        return matches
 
     def __repr__(self):
         return '%s(%s)' % (type(self), self.pattern)
 
+    def __str__(self):
+        return self.pattern_orig
+
 
 def timestamp(s):
     """Convert a --timestamp=s argument to a datetime object"""
@@ -462,18 +512,20 @@ class Location:
     """Object representing a repository / archive location
     """
     proto = user = host = port = path = archive = None
+    # borg mount's FUSE filesystem creates one level of directories from
+    # the archive names. Thus, we must not accept "/" in archive names.
     ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
                         r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
-                        r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
+                        r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
     file_re = re.compile(r'(?P<proto>file)://'
-                         r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
+                         r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
     scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
-                        r'(?P<path>[^:]+)(?:::(?P<archive>.+))?$')
+                        r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
     # get the repo from BORG_RE env and the optional archive from param.
     # if the syntax requires giving REPOSITORY (see "borg mount"),
     # use "::" to let it use the env var.
     # if REPOSITORY argument is optional, it'll automatically use the env.
-    env_re = re.compile(r'(?:::(?P<archive>.+)?)?$')
+    env_re = re.compile(r'(?:::(?P<archive>[^/]+)?)?$')
 
     def __init__(self, text=''):
         self.orig = text

borg/locking.py

@@ -169,6 +169,9 @@ class LockRoster:
             if err.errno != errno.ENOENT:
                 raise
             data = {}
+        except ValueError:
+            # corrupt/empty roster file?
+            data = {}
         return data
 
     def save(self, data):

borg/remote.py

@@ -3,6 +3,7 @@ import fcntl
 import msgpack
 import os
 import select
+import shlex
 from subprocess import Popen, PIPE
 import sys
 import tempfile
@@ -108,8 +109,9 @@ class RepositoryServer:  # pragma: no cover
 
 class RemoteRepository:
     extra_test_args = []
-    remote_path = None
-    umask = None
+    remote_path = 'borg'
+    # default umask, overriden by --umask, defaults to read/write only for owner
+    umask = 0o077
 
     class RPCError(Exception):
         def __init__(self, name):
@@ -125,19 +127,14 @@ class RemoteRepository:
         self.responses = {}
         self.unpacker = msgpack.Unpacker(use_list=False)
         self.p = None
-        # use local umask also for the remote process
-        umask = ['--umask', '%03o' % self.umask]
+        # XXX: ideally, the testsuite would subclass Repository and
+        # override ssh_cmd() instead of this crude hack, although
+        # __testsuite__ is not a valid domain name so this is pretty
+        # safe.
         if location.host == '__testsuite__':
-            args = [sys.executable, '-m', 'borg.archiver', 'serve'] + umask + self.extra_test_args
+            args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args
         else:  # pragma: no cover
-            args = ['ssh']
-            if location.port:
-                args += ['-p', str(location.port)]
-            if location.user:
-                args.append('%s@%s' % (location.user, location.host))
-            else:
-                args.append('%s' % location.host)
-            args += [self.remote_path, 'serve'] + umask
+            args = self.ssh_cmd(location)
         self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE)
         self.stdin_fd = self.p.stdin.fileno()
         self.stdout_fd = self.p.stdout.fileno()
@@ -160,6 +157,21 @@ class RemoteRepository:
     def __repr__(self):
         return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path())
 
+    def umask_flag(self):
+        return ['--umask', '%03o' % self.umask]
+
+    def ssh_cmd(self, location):
+        args = shlex.split(os.environ.get('BORG_RSH', 'ssh'))
+        if location.port:
+            args += ['-p', str(location.port)]
+        if location.user:
+            args.append('%s@%s' % (location.user, location.host))
+        else:
+            args.append('%s' % location.host)
+        # use local umask also for the remote process
+        args += [self.remote_path, 'serve'] + self.umask_flag()
+        return args
+
     def call(self, cmd, *args, **kw):
         for resp in self.call_many(cmd, [args], **kw):
             return resp

borg/repository.py

@@ -301,7 +301,7 @@ class Repository:
             try:
                 objects = list(self.io.iter_objects(segment))
             except IntegrityError as err:
-                report_error('Error reading segment {}: {}'.format(segment, err))
+                report_error(str(err))
                 objects = []
                 if repair:
                     self.io.recover_segment(segment, filename)
@@ -530,30 +530,14 @@ class LoggedIO:
         fd = self.get_fd(segment)
         fd.seek(0)
         if fd.read(MAGIC_LEN) != MAGIC:
-            raise IntegrityError('Invalid segment magic')
+            raise IntegrityError('Invalid segment magic [segment {}, offset {}]'.format(segment, 0))
         offset = MAGIC_LEN
         header = fd.read(self.header_fmt.size)
         while header:
-            try:
-                crc, size, tag = self.header_fmt.unpack(header)
-            except struct.error as err:
-                raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err))
-            if size > MAX_OBJECT_SIZE:
-                raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset))
-            length = size - self.header_fmt.size
-            rest = fd.read(length)
-            if len(rest) != length:
-                raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format(
-                                     offset, length, len(rest)))
-            if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
-                raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset))
-            if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT):
-                raise IntegrityError('Invalid segment entry tag [offset {}]'.format(offset))
-            key = None
-            if tag in (TAG_PUT, TAG_DELETE):
-                key = rest[:32]
+            size, tag, key, data = self._read(fd, self.header_fmt, header, segment, offset,
+                                              (TAG_PUT, TAG_DELETE, TAG_COMMIT))
             if include_data:
-                yield tag, key, offset, rest[32:]
+                yield tag, key, offset, data
             else:
                 yield tag, key, offset
             offset += size
@@ -586,16 +570,44 @@ class LoggedIO:
         fd = self.get_fd(segment)
         fd.seek(offset)
         header = fd.read(self.put_header_fmt.size)
-        crc, size, tag, key = self.put_header_fmt.unpack(header)
-        if size > MAX_OBJECT_SIZE:
-            raise IntegrityError('Invalid segment object size')
-        data = fd.read(size - self.put_header_fmt.size)
-        if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
-            raise IntegrityError('Segment checksum mismatch')
-        if tag != TAG_PUT or id != key:
-            raise IntegrityError('Invalid segment entry header')
+        size, tag, key, data = self._read(fd, self.put_header_fmt, header, segment, offset, (TAG_PUT, ))
+        if id != key:
+            raise IntegrityError('Invalid segment entry header, is not for wanted id [segment {}, offset {}]'.format(
+                segment, offset))
         return data
 
+    def _read(self, fd, fmt, header, segment, offset, acceptable_tags):
+        # some code shared by read() and iter_objects()
+        try:
+            hdr_tuple = fmt.unpack(header)
+        except struct.error as err:
+            raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format(
+                segment, offset, err))
+        if fmt is self.put_header_fmt:
+            crc, size, tag, key = hdr_tuple
+        elif fmt is self.header_fmt:
+            crc, size, tag = hdr_tuple
+            key = None
+        else:
+            raise TypeError("_read called with unsupported format")
+        if size > MAX_OBJECT_SIZE or size < fmt.size:
+            raise IntegrityError('Invalid segment entry size [segment {}, offset {}]'.format(
+                segment, offset))
+        length = size - fmt.size
+        data = fd.read(length)
+        if len(data) != length:
+            raise IntegrityError('Segment entry data short read [segment {}, offset {}]: expected {}, got {} bytes'.format(
+                segment, offset, length, len(data)))
+        if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
+            raise IntegrityError('Segment entry checksum mismatch [segment {}, offset {}]'.format(
+                segment, offset))
+        if tag not in acceptable_tags:
+            raise IntegrityError('Invalid segment entry header, did not get acceptable tag [segment {}, offset {}]'.format(
+                segment, offset))
+        if key is None and tag in (TAG_PUT, TAG_DELETE):
+            key, data = data[:32], data[32:]
+        return size, tag, key, data
+
     def write_put(self, id, data):
         size = len(data) + self.put_header_fmt.size
         fd = self.get_write_fd()

borg/support/__init__.py

@@ -0,0 +1,16 @@
+"""
+3rd party stuff that needed fixing
+
+Note: linux package maintainers feel free to remove any of these hacks
+      IF your python version is not affected.
+
+argparse is broken with default args (double conversion):
+affects: 3.2.0 <= python < 3.2.4
+affects: 3.3.0 <= python < 3.3.1
+
+as we still support 3.2 and 3.3 there is no other way than to bundle
+a fixed version (I just took argparse.py from 3.2.6) and import it from
+here (see import in archiver.py).
+DEPRECATED - remove support.argparse after requiring python 3.4.
+"""
+

borg/support/argparse.py

@@ -0,0 +1,2383 @@
+# Author: Steven J. Bethard <steven.bethard@gmail.com>.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+    - handles both optional and positional arguments
+    - produces highly informative usage messages
+    - supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file::
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    - ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    - ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    - FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    - Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
+        ArgumentDefaultsHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default,
+        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
+        not to change the formatting for help text, and
+        ArgumentDefaultsHelpFormatter adds information about argument defaults
+        to the help.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '1.1'
+__all__ = [
+    'ArgumentParser',
+    'ArgumentError',
+    'ArgumentTypeError',
+    'FileType',
+    'HelpFormatter',
+    'ArgumentDefaultsHelpFormatter',
+    'RawDescriptionHelpFormatter',
+    'RawTextHelpFormatter',
+    'Namespace',
+    'Action',
+    'ONE_OR_MORE',
+    'OPTIONAL',
+    'PARSER',
+    'REMAINDER',
+    'SUPPRESS',
+    'ZERO_OR_MORE',
+]
+
+
+import collections as _collections
+import copy as _copy
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+try:
+    from gettext import gettext, ngettext
+except ImportError:
+    def gettext(message):
+        return message
+    def ngettext(msg1, msg2, n):
+        return msg1 if n == 1 else msg2
+_ = gettext
+
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = 'A...'
+REMAINDER = '...'
+_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format::
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+    """Formatter for generating usage messages and argument help strings.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join([func(*args) for func, args in self.items])
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, actions, groups, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, actions, groups, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+
+            # update the maximum item length
+            invocation_length = max([len(s) for s in invocations])
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_help(self):
+        help = self._root_section.format_help()
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join([part
+                        for part in part_strings
+                        if part and part is not SUPPRESS])
+
+    def _format_usage(self, usage, actions, groups, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if usage is specified, use that
+        if usage is not None:
+            usage = usage % dict(prog=self._prog)
+
+        # if no optionals or positionals are available, usage is just prog
+        elif usage is None and not actions:
+            usage = '%(prog)s' % dict(prog=self._prog)
+
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            prog = '%(prog)s' % dict(prog=self._prog)
+
+            # split optionals from positionals
+            optionals = []
+            positionals = []
+            for action in actions:
+                if action.option_strings:
+                    optionals.append(action)
+                else:
+                    positionals.append(action)
+
+            # build full usage string
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals, groups)
+            usage = ' '.join([s for s in [prog, action_usage] if s])
+
+            # wrap the usage parts if it's too long
+            text_width = self._width - self._current_indent
+            if len(prefix) + len(usage) > text_width:
+
+                # break usage into wrappable parts
+                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                opt_usage = format(optionals, groups)
+                pos_usage = format(positionals, groups)
+                opt_parts = _re.findall(part_regexp, opt_usage)
+                pos_parts = _re.findall(part_regexp, pos_usage)
+                assert ' '.join(opt_parts) == opt_usage
+                assert ' '.join(pos_parts) == pos_usage
+
+                # helper for wrapping lines
+                def get_lines(parts, indent, prefix=None):
+                    lines = []
+                    line = []
+                    if prefix is not None:
+                        line_len = len(prefix) - 1
+                    else:
+                        line_len = len(indent) - 1
+                    for part in parts:
+                        if line_len + 1 + len(part) > text_width:
+                            lines.append(indent + ' '.join(line))
+                            line = []
+                            line_len = len(indent) - 1
+                        line.append(part)
+                        line_len += len(part) + 1
+                    if line:
+                        lines.append(indent + ' '.join(line))
+                    if prefix is not None:
+                        lines[0] = lines[0][len(indent):]
+                    return lines
+
+                # if prog is short, follow it with optionals or positionals
+                if len(prefix) + len(prog) <= 0.75 * text_width:
+                    indent = ' ' * (len(prefix) + len(prog) + 1)
+                    if opt_parts:
+                        lines = get_lines([prog] + opt_parts, indent, prefix)
+                        lines.extend(get_lines(pos_parts, indent))
+                    elif pos_parts:
+                        lines = get_lines([prog] + pos_parts, indent, prefix)
+                    else:
+                        lines = [prog]
+
+                # if prog is long, put it on its own line
+                else:
+                    indent = ' ' * len(prefix)
+                    parts = opt_parts + pos_parts
+                    lines = get_lines(parts, indent)
+                    if len(lines) > 1:
+                        lines = []
+                        lines.extend(get_lines(opt_parts, indent))
+                        lines.extend(get_lines(pos_parts, indent))
+                    lines = [prog] + lines
+
+                # join lines into usage
+                usage = '\n'.join(lines)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions, groups):
+        # find group indices and identify actions in groups
+        group_actions = set()
+        inserts = {}
+        for group in groups:
+            try:
+                start = actions.index(group._group_actions[0])
+            except ValueError:
+                continue
+            else:
+                end = start + len(group._group_actions)
+                if actions[start:end] == group._group_actions:
+                    for action in group._group_actions:
+                        group_actions.add(action)
+                    if not group.required:
+                        if start in inserts:
+                            inserts[start] += ' ['
+                        else:
+                            inserts[start] = '['
+                        inserts[end] = ']'
+                    else:
+                        if start in inserts:
+                            inserts[start] += ' ('
+                        else:
+                            inserts[start] = '('
+                        inserts[end] = ')'
+                    for i in range(start + 1, end):
+                        inserts[i] = '|'
+
+        # collect all actions format strings
+        parts = []
+        for i, action in enumerate(actions):
+
+            # suppressed arguments are marked with None
+            # remove | separators for suppressed arguments
+            if action.help is SUPPRESS:
+                parts.append(None)
+                if inserts.get(i) == '|':
+                    inserts.pop(i)
+                elif inserts.get(i + 1) == '|':
+                    inserts.pop(i + 1)
+
+            # produce all arg strings
+            elif not action.option_strings:
+                part = self._format_args(action, action.dest)
+
+                # if it's in a group, strip the outer []
+                if action in group_actions:
+                    if part[0] == '[' and part[-1] == ']':
+                        part = part[1:-1]
+
+                # add the action string to the list
+                parts.append(part)
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required or in a group
+                if not action.required and action not in group_actions:
+                    part = '[%s]' % part
+
+                # add the action string to the list
+                parts.append(part)
+
+        # insert things at the necessary indices
+        for i in sorted(inserts, reverse=True):
+            parts[i:i] = [inserts[i]]
+
+        # join all the action items with spaces
+        text = ' '.join([item for item in parts if item is not None])
+
+        # clean up separators for mutually exclusive groups
+        open = r'[\[(]'
+        close = r'[\])]'
+        text = _re.sub(r'(%s) ' % open, r'\1', text)
+        text = _re.sub(r' (%s)' % close, r'\1', text)
+        text = _re.sub(r'%s *%s' % (open, close), r'', text)
+        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
+        text = text.strip()
+
+        # return the text
+        return text
+
+    def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            metavar, = self._metavar_formatter(action, action.dest)(1)
+            return metavar
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _metavar_formatter(self, action, default_metavar):
+        if action.metavar is not None:
+            result = action.metavar
+        elif action.choices is not None:
+            choice_strs = [str(choice) for choice in action.choices]
+            result = '{%s}' % ','.join(choice_strs)
+        else:
+            result = default_metavar
+
+        def format(tuple_size):
+            if isinstance(result, tuple):
+                return result
+            else:
+                return (result, ) * tuple_size
+        return format
+
+    def _format_args(self, action, default_metavar):
+        get_metavar = self._metavar_formatter(action, default_metavar)
+        if action.nargs is None:
+            result = '%s' % get_metavar(1)
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % get_metavar(1)
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % get_metavar(2)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % get_metavar(2)
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
+            result = '%s ...' % get_metavar(1)
+        else:
+            formats = ['%s' for _ in range(action.nargs)]
+            result = ' '.join(formats) % get_metavar(action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name in list(params):
+            if params[name] is SUPPRESS:
+                del params[name]
+        for name in list(params):
+            if hasattr(params[name], '__name__'):
+                params[name] = params[name].__name__
+        if params.get('choices') is not None:
+            choices_str = ', '.join([str(c) for c in params['choices']])
+            params['choices'] = choices_str
+        return self._get_help_string(action) % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+    def _get_help_string(self, action):
+        return action.help
+
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    """Help message formatter which retains any formatting in descriptions.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _fill_text(self, text, width, indent):
+        return ''.join([indent + line for line in text.splitlines(True)])
+
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    """Help message formatter which retains formatting of all help text.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+
+class ArgumentDefaultsHelpFormatter(HelpFormatter):
+    """Help message formatter which adds default values to argument help.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _get_help_string(self, action):
+        help = action.help
+        if '%(default)' not in action.help:
+            if action.default is not SUPPRESS:
+                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
+                if action.option_strings or action.nargs in defaulting_nargs:
+                    help += ' (default: %(default)s)'
+        return help
+
+
+# =====================
+# Options and Arguments
+# =====================
+
+def _get_action_name(argument):
+    if argument is None:
+        return None
+    elif argument.option_strings:
+        return  '/'.join(argument.option_strings)
+    elif argument.metavar not in (None, SUPPRESS):
+        return argument.metavar
+    elif argument.dest not in (None, SUPPRESS):
+        return argument.dest
+    else:
+        return None
+
+
+class ArgumentError(Exception):
+    """An error from creating or using an argument (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+
+    def __init__(self, argument, message):
+        self.argument_name = _get_action_name(argument)
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Information about how to convert command line strings to Python objects.
+
+    Action objects are used by an ArgumentParser to represent the information
+    needed to parse a single argument from one or more strings from the
+    command line. The keyword arguments to the Action constructor are also
+    all attributes of Action instances.
+
+    Keyword Arguments:
+
+        - option_strings -- A list of command-line option strings which
+            should be associated with this action.
+
+        - dest -- The name of the attribute to hold the created object(s)
+
+        - nargs -- The number of command-line arguments that should be
+            consumed. By default, one argument will be consumed and a single
+            value will be produced.  Other values include:
+                - N (an integer) consumes N arguments (and produces a list)
+                - '?' consumes zero or one arguments
+                - '*' consumes zero or more arguments (and produces a list)
+                - '+' consumes one or more arguments (and produces a list)
+            Note that the difference between the default and nargs=1 is that
+            with the default, a single value will be produced, while with
+            nargs=1, a list containing a single value will be produced.
+
+        - const -- The value to be produced if the option is specified and the
+            option uses an action that takes no values.
+
+        - default -- The value to be produced if the option is not specified.
+
+        - type -- A callable that accepts a single string argument, and
+            returns the converted value.  The standard Python types str, int,
+            float, and complex are useful examples of such callables.  If None,
+            str is used.
+
+        - choices -- A container of values that should be allowed. If not None,
+            after a command-line argument has been converted to the appropriate
+            type, an exception will be raised if it is not a member of this
+            collection.
+
+        - required -- True if the action must always be specified at the
+            command line. This is only meaningful for optional command-line
+            arguments.
+
+        - help -- The help string describing the argument.
+
+        - metavar -- The name to be used for the option's argument with the
+            help string. If None, the 'dest' value will be used as the name.
+    """
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+
+class _StoreAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for store actions must be > 0; if you '
+                             'have nothing to store, actions such as store '
+                             'true or store const may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
+class _StoreConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+
+class _StoreTrueAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _StoreFalseAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _AppendAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for append actions must be > 0; if arg '
+                             'strings are not supplying the value to append, '
+                             'the append const action may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(values)
+        setattr(namespace, self.dest, items)
+
+
+class _AppendConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(self.const)
+        setattr(namespace, self.dest, items)
+
+
+class _CountAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+
+class _HelpAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+class _VersionAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 version=None,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help="show program's version number and exit"):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+        self.version = version
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
+
+
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+
+        def __init__(self, name, aliases, help):
+            metavar = dest = name
+            if aliases:
+                metavar += ' (%s)' % ', '.join(aliases)
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=dest, help=help,
+                         metavar=metavar)
+
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = _collections.OrderedDict()
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        aliases = kwargs.pop('aliases', ())
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, aliases, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+
+        # make parser available under aliases also
+        for alias in aliases:
+            self._name_parser_map[alias] = parser
+
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            args = {'parser_name': parser_name,
+                    'choices': ', '.join(self._name_parser_map)}
+            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        # store any unrecognized options on the object, so that the top
+        # level parser can decide what to do with them
+        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+        if arg_strings:
+            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
+            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+        - mode -- A string indicating how the file is to be opened. Accepts the
+            same values as the builtin open() function.
+        - bufsize -- The file's desired buffer size. Accepts the same values as
+            the builtin open() function.
+    """
+
+    def __init__(self, mode='r', bufsize=-1):
+        self._mode = mode
+        self._bufsize = bufsize
+
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if 'r' in self._mode:
+                return _sys.stdin
+            elif 'w' in self._mode:
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r') % self._mode
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        try:
+            return open(string, self._mode, self._bufsize)
+        except IOError as e:
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
+
+    def __repr__(self):
+        args = self._mode, self._bufsize
+        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        return '%s(%s)' % (type(self).__name__, args_str)
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+    """Simple object for storing attributes.
+
+    Implements equality by attribute names and values, and provides a simple
+    string representation.
+    """
+
+    def __init__(self, **kwargs):
+        for name in kwargs:
+            setattr(self, name, kwargs[name])
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+
+class _ActionsContainer(object):
+
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._actions = []
+        self._option_string_actions = {}
+
+        # groups
+        self._action_groups = []
+        self._mutually_exclusive_groups = []
+
+        # defaults storage
+        self._defaults = {}
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
+
+        # whether or not there are any optionals that look like negative
+        # numbers -- uses a list so it can be shared and edited
+        self._has_negative_number_optionals = []
+
+    # ====================
+    # Registration methods
+    # ====================
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default accessor methods
+    # ==================================
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action in self._actions:
+            if action.dest in kwargs:
+                action.default = kwargs[action.dest]
+
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
+    # =======================
+    # Adding argument actions
+    # =======================
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        if not callable(action_class):
+            raise ValueError('unknown action "%s"' % (action_class,))
+        action = action_class(**kwargs)
+
+        # raise an error if the action type is not callable
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            raise ValueError('%r is not callable' % (type_func,))
+
+        # raise an error if the metavar does not match the type
+        if hasattr(self, "_get_formatter"):
+            try:
+                self._get_formatter()._format_args(action, None)
+            except TypeError:
+                raise ValueError("length of metavar tuple does not match nargs")
+
+        return self._add_action(action)
+
+    def add_argument_group(self, *args, **kwargs):
+        group = _ArgumentGroup(self, *args, **kwargs)
+        self._action_groups.append(group)
+        return group
+
+    def add_mutually_exclusive_group(self, **kwargs):
+        group = _MutuallyExclusiveGroup(self, **kwargs)
+        self._mutually_exclusive_groups.append(group)
+        return group
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to actions list
+        self._actions.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_string_actions[option_string] = action
+
+        # set the flag if any option strings look like negative numbers
+        for option_string in action.option_strings:
+            if self._negative_number_matcher.match(option_string):
+                if not self._has_negative_number_optionals:
+                    self._has_negative_number_optionals.append(True)
+
+        # return the created action
+        return action
+
+    def _remove_action(self, action):
+        self._actions.remove(action)
+
+    def _add_container_actions(self, container):
+        # collect groups by titles
+        title_group_map = {}
+        for group in self._action_groups:
+            if group.title in title_group_map:
+                msg = _('cannot merge actions - two groups are named %r')
+                raise ValueError(msg % (group.title))
+            title_group_map[group.title] = group
+
+        # map each action to its group
+        group_map = {}
+        for group in container._action_groups:
+
+            # if a group with the title exists, use that, otherwise
+            # create a new group matching the container's group
+            if group.title not in title_group_map:
+                title_group_map[group.title] = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+
+            # map the actions to their new group
+            for action in group._group_actions:
+                group_map[action] = title_group_map[group.title]
+
+        # add container's mutually exclusive groups
+        # NOTE: if add_mutually_exclusive_group ever gains title= and
+        # description= then this code will need to be expanded as above
+        for group in container._mutually_exclusive_groups:
+            mutex_group = self.add_mutually_exclusive_group(
+                required=group.required)
+
+            # map the actions to their new mutex group
+            for action in group._group_actions:
+                group_map[action] = mutex_group
+
+        # add all actions to this container or their group
+        for action in container._actions:
+            group_map.get(action, self)._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+
+        # mark positional arguments as required if at least one is
+        # always required
+        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
+            kwargs['required'] = True
+        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
+            kwargs['required'] = True
+
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                args = {'option': option_string,
+                        'prefix_chars': self.prefix_chars}
+                msg = _('invalid option string %(option)r: '
+                        'must start with a character %(prefix_chars)r')
+                raise ValueError(msg % args)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)
+            if option_string[0] in self.prefix_chars:
+                if len(option_string) > 1:
+                    if option_string[1] in self.prefix_chars:
+                        long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            if not dest:
+                msg = _('dest= is required for options like %r')
+                raise ValueError(msg % option_string)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_string_actions:
+                confl_optional = self._option_string_actions[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = ngettext('conflicting option string: %s',
+                           'conflicting option strings: %s',
+                           len(conflicting_actions))
+        conflict_string = ', '.join([option_string
+                                     for option_string, action
+                                     in conflicting_actions])
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options
+        for option_string, action in conflicting_actions:
+
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_string_actions.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._remove_action(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+
+        # group attributes
+        self.title = title
+        self._group_actions = []
+
+        # share most attributes with the container
+        self._registries = container._registries
+        self._actions = container._actions
+        self._option_string_actions = container._option_string_actions
+        self._defaults = container._defaults
+        self._has_negative_number_optionals = \
+            container._has_negative_number_optionals
+        self._mutually_exclusive_groups = container._mutually_exclusive_groups
+
+    def _add_action(self, action):
+        action = super(_ArgumentGroup, self)._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        super(_ArgumentGroup, self)._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class _MutuallyExclusiveGroup(_ArgumentGroup):
+
+    def __init__(self, container, required=False):
+        super(_MutuallyExclusiveGroup, self).__init__(container)
+        self.required = required
+        self._container = container
+
+    def _add_action(self, action):
+        if action.required:
+            msg = _('mutually exclusive arguments must be optional')
+            raise ValueError(msg)
+        action = self._container._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        self._container._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+    """Object for parsing command line strings into Python objects.
+
+    Keyword Arguments:
+        - prog -- The name of the program (default: sys.argv[0])
+        - usage -- A usage message (default: auto-generated from arguments)
+        - description -- A description of what the program does
+        - epilog -- Text following the argument descriptions
+        - parents -- Parsers whose arguments should be copied into this one
+        - formatter_class -- HelpFormatter class for printing help messages
+        - prefix_chars -- Characters that prefix optional arguments
+        - fromfile_prefix_chars -- Characters that prefix files containing
+            additional arguments
+        - argument_default -- The default value for all arguments
+        - conflict_handler -- String indicating how to handle conflicts
+        - add_help -- Add a -h/-help option
+    """
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 fromfile_prefix_chars=None,
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        if version is not None:
+            import warnings
+            warnings.warn(
+                """The "version" argument to ArgumentParser is deprecated. """
+                """Please use """
+                """"add_argument(..., action='version', version="N", ...)" """
+                """instead""", DeprecationWarning)
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.fromfile_prefix_chars = fromfile_prefix_chars
+        self.add_help = add_help
+
+        add_group = self.add_argument_group
+        self._positionals = add_group(_('positional arguments'))
+        self._optionals = add_group(_('optional arguments'))
+        self._subparsers = None
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if self.add_help:
+            self.add_argument(
+                default_prefix+'h', default_prefix*2+'help',
+                action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                default_prefix+'v', default_prefix*2+'version',
+                action='version', default=SUPPRESS,
+                version=self.version,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+    def add_subparsers(self, **kwargs):
+        if self._subparsers is not None:
+            self.error(_('cannot have multiple subparser arguments'))
+
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        if 'title' in kwargs or 'description' in kwargs:
+            title = _(kwargs.pop('title', 'subcommands'))
+            description = _(kwargs.pop('description', None))
+            self._subparsers = self.add_argument_group(title, description)
+        else:
+            self._subparsers = self._positionals
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            positionals = self._get_positional_actions()
+            groups = self._mutually_exclusive_groups
+            formatter.add_usage(self.usage, positionals, groups, '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._subparsers._add_action(action)
+
+        # return the created parsers action
+        return action
+
+    def _add_action(self, action):
+        if action.option_strings:
+            self._optionals._add_action(action)
+        else:
+            self._positionals._add_action(action)
+        return action
+
+    def _get_optional_actions(self):
+        return [action
+                for action in self._actions
+                if action.option_strings]
+
+    def _get_positional_actions(self):
+        return [action
+                for action in self._actions
+                if not action.option_strings]
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+    def parse_args(self, args=None, namespace=None):
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = _('unrecognized arguments: %s')
+            self.error(msg % ' '.join(argv))
+        return args
+
+    def parse_known_args(self, args=None, namespace=None):
+        if args is None:
+            # args default to the system args
+            args = _sys.argv[1:]
+        else:
+            # make sure that args are mutable
+            args = list(args)
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+
+        # add any action defaults that aren't present
+        for action in self._actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        setattr(namespace, action.dest, action.default)
+
+        # add any parser defaults that aren't present
+        for dest in self._defaults:
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, self._defaults[dest])
+
+        # parse the arguments and exit if there are any errors
+        try:
+            namespace, args = self._parse_known_args(args, namespace)
+            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+            return namespace, args
+        except ArgumentError:
+            err = _sys.exc_info()[1]
+            self.error(str(err))
+
+    def _parse_known_args(self, arg_strings, namespace):
+        # replace arg strings that are file references
+        if self.fromfile_prefix_chars is not None:
+            arg_strings = self._read_args_from_files(arg_strings)
+
+        # map all mutually exclusive arguments to the other arguments
+        # they can't occur with
+        action_conflicts = {}
+        for mutex_group in self._mutually_exclusive_groups:
+            group_actions = mutex_group._group_actions
+            for i, mutex_action in enumerate(mutex_group._group_actions):
+                conflicts = action_conflicts.setdefault(mutex_action, [])
+                conflicts.extend(group_actions[:i])
+                conflicts.extend(group_actions[i + 1:])
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        seen_actions = set()
+        seen_non_default_actions = set()
+
+        def take_action(action, argument_strings, option_string=None):
+            seen_actions.add(action)
+            argument_values = self._get_values(action, argument_strings)
+
+            # error if this argument is not allowed with other previously
+            # seen arguments, assuming that actions that use the default
+            # value don't really count as "present"
+            if argument_values is not action.default:
+                seen_non_default_actions.add(action)
+                for conflict_action in action_conflicts.get(action, []):
+                    if conflict_action in seen_non_default_actions:
+                        msg = _('not allowed with argument %s')
+                        action_name = _get_action_name(conflict_action)
+                        raise ArgumentError(action, msg % action_name)
+
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+
+            # get the optional identified at this index
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # identify additional optionals in the same arg string
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, skip it
+                if action is None:
+                    extras.append(arg_strings[start_index])
+                    return start_index + 1
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        char = option_string[0]
+                        option_string = char + explicit_arg[0]
+                        new_explicit_arg = explicit_arg[1:] or None
+                        optionals_map = self._option_string_actions
+                        if option_string in optionals_map:
+                            action = optionals_map[option_string]
+                            explicit_arg = new_explicit_arg
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        extras = []
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+
+            # consume any Positionals preceding the next option
+            next_option_string_index = min([
+                index
+                for index in option_string_indices
+                if index >= start_index])
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were extra arguments
+            if start_index not in option_string_indices:
+                strings = arg_strings[start_index:next_option_string_index]
+                extras.extend(strings)
+                start_index = next_option_string_index
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were extras
+        extras.extend(arg_strings[stop_index:])
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present, and convert defaults.
+        for action in self._actions:
+            if action not in seen_actions:
+                if action.required:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+                else:
+                    # Convert action default now instead of doing it before
+                    # parsing arguments to avoid calling convert functions
+                    # twice (which may fail) if the argument was given, but
+                    # only if it was defined already in the namespace
+                    if (action.default is not None and
+                            isinstance(action.default, str) and
+                            hasattr(namespace, action.dest) and
+                            action.default is getattr(namespace, action.dest)):
+                        setattr(namespace, action.dest,
+                                self._get_value(action, action.default))
+
+        # make sure all required groups had one option present
+        for group in self._mutually_exclusive_groups:
+            if group.required:
+                for action in group._group_actions:
+                    if action in seen_non_default_actions:
+                        break
+
+                # if no actions were used, report the error
+                else:
+                    names = [_get_action_name(action)
+                             for action in group._group_actions
+                             if action.help is not SUPPRESS]
+                    msg = _('one of the arguments %s is required')
+                    self.error(msg % ' '.join(names))
+
+        # return the updated namespace and the extra arguments
+        return namespace, extras
+
+    def _read_args_from_files(self, arg_strings):
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+
+            # for regular arguments, just add them back into the list
+            if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+
+            # replace arguments referencing files with the file content
+            else:
+                try:
+                    args_file = open(arg_string[1:])
+                    try:
+                        arg_strings = []
+                        for arg_line in args_file.read().splitlines():
+                            for arg in self.convert_arg_line_to_args(arg_line):
+                                arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                    finally:
+                        args_file.close()
+                except IOError:
+                    err = _sys.exc_info()[1]
+                    self.error(str(err))
+
+        # return the modified argument list
+        return new_arg_strings
+
+    def convert_arg_line_to_args(self, arg_line):
+        return [arg_line]
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match
+        if match is None:
+            nargs_errors = {
+                None: _('expected one argument'),
+                OPTIONAL: _('expected at most one argument'),
+                ONE_OR_MORE: _('expected at least one argument'),
+            }
+            default = ngettext('expected %s argument',
+                               'expected %s arguments',
+                               action.nargs) % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in range(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join([self._get_nargs_pattern(action)
+                               for action in actions_slice])
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend([len(string) for string in match.groups()])
+                break
+
+        # return the list of arg string counts
+        return result
+
+    def _parse_optional(self, arg_string):
+        # if it's an empty string, it was meant to be a positional
+        if not arg_string:
+            return None
+
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_string_actions:
+            action = self._option_string_actions[arg_string]
+            return action, arg_string, None
+
+        # if it's just a single character, it was meant to be positional
+        if len(arg_string) == 1:
+            return None
+
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = self._get_option_tuples(arg_string)
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join([option_string
+                for action, option_string, explicit_arg in option_tuples])
+            args = {'option': arg_string, 'matches': options}
+            msg = _('ambiguous option: %(option)s could match %(matches)s')
+            self.error(msg % args)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        # unless there are negative-number-like options
+        if self._negative_number_matcher.match(arg_string):
+            if not self._has_negative_number_optionals:
+                return None
+
+        # if it contains a space, it was meant to be a positional
+        if ' ' in arg_string:
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_tuples(self, option_string):
+        result = []
+
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            for option_string in self._option_string_actions:
+                if option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # single character options can be concatenated with their arguments
+        # but multiple character options always have to have their argument
+        # separate
+        elif option_string[0] in chars and option_string[1] not in chars:
+            option_prefix = option_string
+            explicit_arg = None
+            short_option_prefix = option_string[:2]
+            short_explicit_arg = option_string[2:]
+
+            for option_string in self._option_string_actions:
+                if option_string == short_option_prefix:
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, short_explicit_arg
+                    result.append(tup)
+                elif option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # shouldn't ever get here
+        else:
+            self.error(_('unexpected option string: %s') % option_string)
+
+        # return the collected option tuples
+        return result
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs == PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER, REMAINDER args, strip out first '--'
+        if action.nargs not in [PARSER, REMAINDER]:
+            try:
+                arg_strings.remove('--')
+            except ValueError:
+                pass
+
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, str):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs == PARSER:
+            value = [self._get_value(action, v) for v in arg_strings]
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = [self._get_value(action, v) for v in arg_strings]
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            args = {'type': name, 'value': arg_string}
+            msg = _('invalid %(type)s value: %(value)r')
+            raise ArgumentError(action, msg % args)
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            args = {'value': value,
+                    'choices': ', '.join(map(repr, action.choices))}
+            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+            raise ArgumentError(action, msg % args)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+
+        # description
+        formatter.add_text(self.description)
+
+        # positionals, optionals and user-defined groups
+        for action_group in self._action_groups:
+            formatter.start_section(action_group.title)
+            formatter.add_text(action_group.description)
+            formatter.add_arguments(action_group._group_actions)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        import warnings
+        warnings.warn(
+            'The format_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+    def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        import warnings
+        warnings.warn(
+            'The print_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+    # ===============
+    # Exiting methods
+    # ===============
+    def exit(self, status=0, message=None):
+        if message:
+            self._print_message(message, _sys.stderr)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        args = {'prog': self.prog, 'message': message}
+        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)

borg/testsuite/__init__.py

@@ -2,6 +2,7 @@ from contextlib import contextmanager
 import filecmp
 import os
 import posix
+import stat
 import sys
 import sysconfig
 import time
@@ -27,6 +28,8 @@ elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
 else:
     st_mtime_ns_round = -9
 
+if sys.platform.startswith('netbsd'):
+    st_mtime_ns_round = -4  # only >1 microsecond resolution here?
 
 has_mtime_ns = sys.version >= '3.3'
 utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
@@ -72,6 +75,11 @@ class BaseTestCase(unittest.TestCase):
                 attrs.append('st_nlink')
             d1 = [filename] + [getattr(s1, a) for a in attrs]
             d2 = [filename] + [getattr(s2, a) for a in attrs]
+            # ignore st_rdev if file is not a block/char device, fixes #203
+            if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]):
+                d1[4] = None
+            if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]):
+                d2[4] = None
             if not os.path.islink(path1) or utime_supports_fd:
                 # Older versions of llfuse do not support ns precision properly
                 if fuse and not have_fuse_mtime_ns:

borg/testsuite/archiver.py

@@ -162,7 +162,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         # Directory
         self.create_regular_file('dir2/file2', size=1024 * 80)
         # File mode
-        os.chmod('input/file1', 0o7755)
+        os.chmod('input/file1', 0o4755)
         # Hard link
         os.link(os.path.join(self.input_path, 'file1'),
                 os.path.join(self.input_path, 'hardlink'))
@@ -264,7 +264,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         st = os.stat(filename)
         self.assert_equal(st.st_size, total_len)
         if sparse_support and hasattr(st, 'st_blocks'):
-            self.assert_true(st.st_blocks * 512 < total_len / 10)  # is input sparse?
+            self.assert_true(st.st_blocks * 512 < total_len / 9)  # is input sparse?
         self.cmd('init', self.repository_location)
         self.cmd('create', self.repository_location + '::test', 'input')
         with changedir('output'):
@@ -279,7 +279,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         st = os.stat(filename)
         self.assert_equal(st.st_size, total_len)
         if sparse_support and hasattr(st, 'st_blocks'):
-            self.assert_true(st.st_blocks * 512 < total_len / 10)  # is output sparse?
+            self.assert_true(st.st_blocks * 512 < total_len / 9)  # is output sparse?
 
     def test_unusual_filenames(self):
         filenames = ['normal', 'with some blanks', '(with_parens)', ]

borg/testsuite/compress.py

@@ -93,7 +93,7 @@ def test_compressor():
         params_list += [
             dict(name='lzma', level=0, buffer=buffer),
             dict(name='lzma', level=6, buffer=buffer),
-            dict(name='lzma', level=9, buffer=buffer),
+            # we do not test lzma on level 9 because of the huge memory needs
         ]
     for params in params_list:
         c = Compressor(**params)

borg/testsuite/helpers.py

@@ -1,12 +1,14 @@
 import hashlib
 from time import mktime, strptime
 from datetime import datetime, timezone, timedelta
+import os
 
 import pytest
+import sys
 import msgpack
 
-from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, ExcludePattern, make_path_safe, \
-    prune_within, prune_split, \
+from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \
+    prune_within, prune_split, get_cache_dir, \
     StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams
 from . import BaseTestCase
 
@@ -80,6 +82,11 @@ class TestLocationWithoutEnv:
         with pytest.raises(ValueError):
             Location('ssh://localhost:22/path:archive')
 
+    def test_no_slashes(self, monkeypatch):
+        monkeypatch.delenv('BORG_REPO', raising=False)
+        with pytest.raises(ValueError):
+            Location('/some/path/to/repo::archive_name_with/slashes/is_invalid')
+
     def test_canonical_path(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
         locations = ['some/path::archive', 'file://some/path::archive', 'host:some/path::archive',
@@ -133,6 +140,11 @@ class TestLocationWithEnv:
         assert repr(Location()) == \
                "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)"
 
+    def test_no_slashes(self, monkeypatch):
+        monkeypatch.setenv('BORG_REPO', '/some/absolute/path')
+        with pytest.raises(ValueError):
+            Location('::archive_name_with/slashes/is_invalid')
+
 
 class FormatTimedeltaTestCase(BaseTestCase):
 
@@ -178,6 +190,72 @@ class PatternTestCase(BaseTestCase):
                           ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg'])
 
 
+@pytest.mark.skipif(sys.platform in ('darwin',), reason='all but OS X test')
+class PatternNonAsciiTestCase(BaseTestCase):
+    def testComposedUnicode(self):
+        pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}'
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert not i.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+        assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert not e.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+
+    def testDecomposedUnicode(self):
+        pattern = 'ba\N{COMBINING ACUTE ACCENT}'
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert not i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+        assert not e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+    
+    def testInvalidUnicode(self):
+        pattern = str(b'ba\x80', 'latin1')
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert not i.match("ba/foo")
+        assert i.match(str(b"ba\x80/foo", 'latin1'))
+        assert not e.match("ba/foo")
+        assert e.match(str(b"ba\x80/foo", 'latin1'))
+
+
+@pytest.mark.skipif(sys.platform not in ('darwin',), reason='OS X test')
+class OSXPatternNormalizationTestCase(BaseTestCase):
+    def testComposedUnicode(self):
+        pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}'
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+        assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+    
+    def testDecomposedUnicode(self):
+        pattern = 'ba\N{COMBINING ACUTE ACCENT}'
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+        assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo")
+        assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo")
+    
+    def testInvalidUnicode(self):
+        pattern = str(b'ba\x80', 'latin1')
+        i = IncludePattern(pattern)
+        e = ExcludePattern(pattern)
+
+        assert not i.match("ba/foo")
+        assert i.match(str(b"ba\x80/foo", 'latin1'))
+        assert not e.match("ba/foo")
+        assert e.match(str(b"ba\x80/foo", 'latin1'))
+
+
 def test_compression_specs():
     with pytest.raises(ValueError):
         CompressionSpec('')
@@ -304,3 +382,20 @@ class TestParseTimestamp(BaseTestCase):
     def test(self):
         self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc))
         self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc))
+
+
+def test_get_cache_dir():
+    """test that get_cache_dir respects environement"""
+    # reset BORG_CACHE_DIR in order to test default
+    old_env = None
+    if os.environ.get('BORG_CACHE_DIR'):
+        old_env = os.environ['BORG_CACHE_DIR']
+        del(os.environ['BORG_CACHE_DIR'])
+    assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg')
+    os.environ['XDG_CACHE_HOME'] = '/var/tmp/.cache'
+    assert get_cache_dir() == os.path.join('/var/tmp/.cache', 'borg')
+    os.environ['BORG_CACHE_DIR'] = '/var/tmp'
+    assert get_cache_dir() == '/var/tmp'
+    # reset old env
+    if old_env is not None:
+        os.environ['BORG_CACHE_DIR'] = old_env

borg/testsuite/repository.py

@@ -325,6 +325,15 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
     def test_invalid_rpc(self):
         self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', None))
 
+    def test_ssh_cmd(self):
+        assert self.repository.umask is not None
+        assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag()
+        os.environ['BORG_RSH'] = 'ssh --foo'
+        assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com', 'borg', 'serve'] + self.repository.umask_flag()
+
 
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
 

borg/testsuite/upgrader.py

@@ -0,0 +1,163 @@
+import os
+import shutil
+import tempfile
+
+import pytest
+
+try:
+    import attic.repository
+    import attic.key
+    import attic.helpers
+except ImportError:
+    attic = None
+
+from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
+from ..helpers import get_keys_dir
+from ..key import KeyfileKey
+from ..repository import Repository, MAGIC
+
+pytestmark = pytest.mark.skipif(attic is None,
+                                reason='cannot find an attic install')
+
+
+def repo_valid(path):
+    """
+    utility function to check if borg can open a repository
+
+    :param path: the path to the repository
+    :returns: if borg can check the repository
+    """
+    repository = Repository(str(path), create=False)
+    # can't check raises() because check() handles the error
+    state = repository.check()
+    repository.close()
+    return state
+
+
+def key_valid(path):
+    """
+    check that the new keyfile is alright
+
+    :param path: the path to the key file
+    :returns: if the file starts with the borg magic string
+    """
+    keyfile = os.path.join(get_keys_dir(),
+                           os.path.basename(path))
+    with open(keyfile, 'r') as f:
+        return f.read().startswith(KeyfileKey.FILE_ID)
+
+
+@pytest.fixture()
+def attic_repo(tmpdir):
+    """
+    create an attic repo with some stuff in it
+
+    :param tmpdir: path to the repository to be created
+    :returns: a attic.repository.Repository object
+    """
+    attic_repo = attic.repository.Repository(str(tmpdir), create=True)
+    # throw some stuff in that repo, copied from `RepositoryTestCase.test1`
+    for x in range(100):
+        attic_repo.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
+    attic_repo.commit()
+    attic_repo.close()
+    return attic_repo
+
+
+def test_convert_segments(tmpdir, attic_repo):
+    """test segment conversion
+
+    this will load the given attic repository, list all the segments
+    then convert them one at a time. we need to close the repo before
+    conversion otherwise we have errors from borg
+
+    :param tmpdir: a temporary directory to run the test in (builtin
+    fixture)
+    :param attic_repo: a populated attic repository (fixture)
+    """
+    # check should fail because of magic number
+    assert not repo_valid(tmpdir)
+    print("opening attic repository with borg and converting")
+    repo = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    segments = [filename for i, filename in repo.io.segment_iterator()]
+    repo.close()
+    repo.convert_segments(segments, dryrun=False)
+    repo.convert_cache(dryrun=False)
+    assert repo_valid(tmpdir)
+
+
+class MockArgs:
+    """
+    mock attic location
+
+    this is used to simulate a key location with a properly loaded
+    repository object to create a key file
+    """
+    def __init__(self, path):
+        self.repository = attic.helpers.Location(path)
+
+
+@pytest.fixture()
+def attic_key_file(attic_repo, tmpdir):
+    """
+    create an attic key file from the given repo, in the keys
+    subdirectory of the given tmpdir
+
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :returns: the KeyfileKey object as returned by
+    attic.key.KeyfileKey.create()
+    """
+    keys_dir = str(tmpdir.mkdir('keys'))
+
+    # we use the repo dir for the created keyfile, because we do
+    # not want to clutter existing keyfiles
+    os.environ['ATTIC_KEYS_DIR'] = keys_dir
+
+    # we use the same directory for the converted files, which
+    # will clutter the previously created one, which we don't care
+    # about anyways. in real runs, the original key will be retained.
+    os.environ['BORG_KEYS_DIR'] = keys_dir
+    os.environ['ATTIC_PASSPHRASE'] = 'test'
+    return attic.key.KeyfileKey.create(attic_repo,
+                                       MockArgs(keys_dir))
+
+
+def test_keys(tmpdir, attic_repo, attic_key_file):
+    """test key conversion
+
+    test that we can convert the given key to a properly formatted
+    borg key. assumes that the ATTIC_KEYS_DIR and BORG_KEYS_DIR have
+    been properly populated by the attic_key_file fixture.
+
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    """
+    repository = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    keyfile = AtticKeyfileKey.find_key_file(repository)
+    AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False)
+    assert key_valid(attic_key_file.path)
+
+
+def test_convert_all(tmpdir, attic_repo, attic_key_file):
+    """test all conversion steps
+
+    this runs everything. mostly redundant test, since everything is
+    done above. yet we expect a NotImplementedError because we do not
+    convert caches yet.
+
+    :param tmpdir: a temporary directory (a builtin fixture)
+    :param attic_repo: an attic.repository.Repository object (fixture
+    define above)
+    :param attic_key_file: an attic.key.KeyfileKey (fixture created above)
+    """
+    # check should fail because of magic number
+    assert not repo_valid(tmpdir)
+    print("opening attic repository with borg and converting")
+    repo = AtticRepositoryUpgrader(str(tmpdir), create=False)
+    repo.upgrade(dryrun=False)
+    assert key_valid(attic_key_file.path)
+    assert repo_valid(tmpdir)

borg/testsuite/xattr.py

@@ -17,17 +17,23 @@ class XattrTestCase(BaseTestCase):
     def tearDown(self):
         os.unlink(self.symlink)
 
+    def assert_equal_se(self, is_x, want_x):
+        # check 2 xattr lists for equality, but ignore security.selinux attr
+        is_x = set(is_x) - {'security.selinux'}
+        want_x = set(want_x)
+        self.assert_equal(is_x, want_x)
+
     def test(self):
-        self.assert_equal(listxattr(self.tmpfile.name), [])
-        self.assert_equal(listxattr(self.tmpfile.fileno()), [])
-        self.assert_equal(listxattr(self.symlink), [])
+        self.assert_equal_se(listxattr(self.tmpfile.name), [])
+        self.assert_equal_se(listxattr(self.tmpfile.fileno()), [])
+        self.assert_equal_se(listxattr(self.symlink), [])
         setxattr(self.tmpfile.name, 'user.foo', b'bar')
         setxattr(self.tmpfile.fileno(), 'user.bar', b'foo')
         setxattr(self.tmpfile.name, 'user.empty', None)
-        self.assert_equal(set(listxattr(self.tmpfile.name)), set(['user.foo', 'user.bar', 'user.empty']))
-        self.assert_equal(set(listxattr(self.tmpfile.fileno())), set(['user.foo', 'user.bar', 'user.empty']))
-        self.assert_equal(set(listxattr(self.symlink)), set(['user.foo', 'user.bar', 'user.empty']))
-        self.assert_equal(listxattr(self.symlink, follow_symlinks=False), [])
+        self.assert_equal_se(listxattr(self.tmpfile.name), ['user.foo', 'user.bar', 'user.empty'])
+        self.assert_equal_se(listxattr(self.tmpfile.fileno()), ['user.foo', 'user.bar', 'user.empty'])
+        self.assert_equal_se(listxattr(self.symlink), ['user.foo', 'user.bar', 'user.empty'])
+        self.assert_equal_se(listxattr(self.symlink, follow_symlinks=False), [])
         self.assert_equal(getxattr(self.tmpfile.name, 'user.foo'), b'bar')
         self.assert_equal(getxattr(self.tmpfile.fileno(), 'user.foo'), b'bar')
         self.assert_equal(getxattr(self.symlink, 'user.foo'), b'bar')

borg/upgrader.py

@@ -0,0 +1,233 @@
+from binascii import hexlify
+import os
+import shutil
+import time
+
+from .helpers import get_keys_dir, get_cache_dir
+from .locking import UpgradableLock
+from .repository import Repository, MAGIC
+from .key import KeyfileKey, KeyfileNotFoundError
+
+ATTIC_MAGIC = b'ATTICSEG'
+
+
+class AtticRepositoryUpgrader(Repository):
+    def upgrade(self, dryrun=True):
+        """convert an attic repository to a borg repository
+
+        those are the files that need to be upgraded here, from most
+        important to least important: segments, key files, and various
+        caches, the latter being optional, as they will be rebuilt if
+        missing.
+
+        we nevertheless do the order in reverse, as we prefer to do
+        the fast stuff first, to improve interactivity.
+        """
+        print("reading segments from attic repository using borg")
+        # we need to open it to load the configuration and other fields
+        self.open(self.path, exclusive=False)
+        segments = [filename for i, filename in self.io.segment_iterator()]
+        try:
+            keyfile = self.find_attic_keyfile()
+        except KeyfileNotFoundError:
+            print("no key file found for repository")
+        else:
+            self.convert_keyfiles(keyfile, dryrun)
+        self.close()
+        # partial open: just hold on to the lock
+        self.lock = UpgradableLock(os.path.join(self.path, 'lock'),
+                                   exclusive=True).acquire()
+        try:
+            self.convert_cache(dryrun)
+            self.convert_segments(segments, dryrun)
+        finally:
+            self.lock.release()
+            self.lock = None
+
+    @staticmethod
+    def convert_segments(segments, dryrun):
+        """convert repository segments from attic to borg
+
+        replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in
+        `$ATTIC_REPO/data/**`.
+
+        luckily the magic string length didn't change so we can just
+        replace the 8 first bytes of all regular files in there."""
+        print("converting %d segments..." % len(segments))
+        i = 0
+        for filename in segments:
+            i += 1
+            print("\rconverting segment %d/%d in place, %.2f%% done (%s)"
+                  % (i, len(segments), 100*float(i)/len(segments), filename), end='')
+            if dryrun:
+                time.sleep(0.001)
+            else:
+                AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC)
+        print()
+
+    @staticmethod
+    def header_replace(filename, old_magic, new_magic):
+        with open(filename, 'r+b') as segment:
+            segment.seek(0)
+            # only write if necessary
+            if segment.read(len(old_magic)) == old_magic:
+                segment.seek(0)
+                segment.write(new_magic)
+
+    def find_attic_keyfile(self):
+        """find the attic keyfiles
+
+        the keyfiles are loaded by `KeyfileKey.find_key_file()`. that
+        finds the keys with the right identifier for the repo.
+
+        this is expected to look into $HOME/.attic/keys or
+        $ATTIC_KEYS_DIR for key files matching the given Borg
+        repository.
+
+        it is expected to raise an exception (KeyfileNotFoundError) if
+        no key is found. whether that exception is from Borg or Attic
+        is unclear.
+
+        this is split in a separate function in case we want to use
+        the attic code here directly, instead of our local
+        implementation."""
+        return AtticKeyfileKey.find_key_file(self)
+
+    @staticmethod
+    def convert_keyfiles(keyfile, dryrun):
+
+        """convert key files from attic to borg
+
+        replacement pattern is `s/ATTIC KEY/BORG_KEY/` in
+        `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or
+        `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or
+        `$HOME/.borg/keys`.
+
+        no need to decrypt to convert. we need to rewrite the whole
+        key file because magic string length changed, but that's not a
+        problem because the keyfiles are small (compared to, say,
+        all the segments)."""
+        print("converting keyfile %s" % keyfile)
+        with open(keyfile, 'r') as f:
+            data = f.read()
+        data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1)
+        keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile))
+        print("writing borg keyfile to %s" % keyfile)
+        if not dryrun:
+            with open(keyfile, 'w') as f:
+                f.write(data)
+
+    def convert_cache(self, dryrun):
+        """convert caches from attic to borg
+
+        those are all hash indexes, so we need to
+        `s/ATTICIDX/BORG_IDX/` in a few locations:
+
+        * the repository index (in `$ATTIC_REPO/index.%d`, where `%d`
+          is the `Repository.get_index_transaction_id()`), which we
+          should probably update, with a lock, see
+          `Repository.open()`, which i'm not sure we should use
+          because it may write data on `Repository.close()`...
+
+        * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or
+          `$HOME/.cache/attic/<repoid>/`), which we could just drop,
+          but if we'd want to convert, we could open it with the
+          `Cache.open()`, edit in place and then `Cache.close()` to
+          make sure we have locking right
+        """
+        caches = []
+        transaction_id = self.get_index_transaction_id()
+        if transaction_id is None:
+            print('no index file found for repository %s' % self.path)
+        else:
+            caches += [os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8')]
+
+        # copy of attic's get_cache_dir()
+        attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR',
+                                         os.path.join(os.path.expanduser('~'),
+                                                      '.cache', 'attic'))
+        attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii'))
+        borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii'))
+
+        def copy_cache_file(path):
+            """copy the given attic cache path into the borg directory
+
+            does nothing if dryrun is True. also expects
+            attic_cache_dir and borg_cache_dir to be set in the parent
+            scope, to the directories path including the repository
+            identifier.
+
+            :params path: the basename of the cache file to copy
+            (example: "files" or "chunks") as a string
+
+            :returns: the borg file that was created or None if non
+            was created.
+
+            """
+            attic_file = os.path.join(attic_cache_dir, path)
+            if os.path.exists(attic_file):
+                borg_file = os.path.join(borg_cache_dir, path)
+                if os.path.exists(borg_file):
+                    print("borg cache file already exists in %s, skipping conversion of %s" % (borg_file, attic_file))
+                else:
+                    print("copying attic cache file from %s to %s" % (attic_file, borg_file))
+                    if not dryrun:
+                        shutil.copyfile(attic_file, borg_file)
+                    return borg_file
+            else:
+                print("no %s cache file found in %s" % (path, attic_file))
+            return None
+
+        # XXX: untested, because generating cache files is a PITA, see
+        # Archiver.do_create() for proof
+        if os.path.exists(attic_cache_dir):
+            if not os.path.exists(borg_cache_dir):
+                os.makedirs(borg_cache_dir)
+
+            # file that we don't have a header to convert, just copy
+            for cache in ['config', 'files']:
+                copy_cache_file(cache)
+
+            # we need to convert the headers of those files, copy first
+            for cache in ['chunks']:
+                copied = copy_cache_file(cache)
+                if copied:
+                    print("converting cache %s" % cache)
+                    if not dryrun:
+                        AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX')
+
+
+class AtticKeyfileKey(KeyfileKey):
+    """backwards compatible Attic key file parser"""
+    FILE_ID = 'ATTIC KEY'
+
+    # verbatim copy from attic
+    @staticmethod
+    def get_keys_dir():
+        """Determine where to repository keys and cache"""
+        return os.environ.get('ATTIC_KEYS_DIR',
+                              os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
+
+    @classmethod
+    def find_key_file(cls, repository):
+        """copy of attic's `find_key_file`_
+
+        this has two small modifications:
+
+        1. it uses the above `get_keys_dir`_ instead of the global one,
+           assumed to be borg's
+
+        2. it uses `repository.path`_ instead of
+           `repository._location.canonical_path`_ because we can't
+           assume the repository has been opened by the archiver yet
+        """
+        get_keys_dir = cls.get_keys_dir
+        id = hexlify(repository.id).decode('ascii')
+        keys_dir = get_keys_dir()
+        for name in os.listdir(keys_dir):
+            filename = os.path.join(keys_dir, name)
+            with open(filename, 'r') as fd:
+                line = fd.readline().strip()
+                if line and line.startswith(cls.FILE_ID) and line[10:] == id:
+                    return filename
+        raise KeyfileNotFoundError(repository.path, get_keys_dir())

docs/Makefile

@@ -36,8 +36,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 
-html:
-	./update_usage.sh
+html: usage api.rst
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@@ -140,3 +139,32 @@ gh-io: html
 
 inotify: html
 	while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done
+
+# generate list of targets
+usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//')
+
+# generate help file based on usage
+usage/%.rst.inc: ../borg/archiver.py
+	@echo generating usage for $*
+	@printf ".. _borg_$*:\n\n" > $@
+	@printf "borg $*\n" >> $@
+	@echo -n borg $* | tr 'a-z- ' '-' >> $@
+	@printf "\n::\n\n" >> $@
+	@borg help $* --usage-only | sed -e 's/^/    /' >> $@
+	@printf "\nDescription\n~~~~~~~~~~~\n" >> $@
+	@borg help $* --epilog-only >> $@
+
+api.rst: Makefile
+	@echo "auto-generating API documentation"
+	@echo "Borg Backup API documentation" > $@
+	@echo "=============================" >> $@
+	@echo "" >> $@
+	@for mod in ../borg/*.pyx ../borg/*.py; do \
+		if echo "$$mod" | grep -q "/_"; then \
+			continue ; \
+		fi ; \
+		printf ".. automodule:: "; \
+		echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \
+		echo "    :members:"; \
+		echo "    :undoc-members:"; \
+	done >> $@

docs/_themes/local/sidebarusefullinks.html

@@ -5,7 +5,7 @@
 <ul>
   <li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
   <li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
-  <li><a href="https://github.com/borgbackup/borg/issues/147">Binary Packages</a></li>
+  <li><a href="https://github.com/borgbackup/borg/issues/214">Binaries</a></li>
   <li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>
   <li><a href="https://github.com/borgbackup/borg">GitHub</a></li>
   <li><a href="https://github.com/borgbackup/borg/issues">Issue Tracker</a></li>

docs/conf.py

@@ -218,7 +218,7 @@ latex_documents = [
 #     ['see "AUTHORS" file'], 1)
 #]
 
-extensions = ['sphinx.ext.extlinks']
+extensions = ['sphinx.ext.extlinks', 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode']
 
 extlinks = {
     'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),

docs/development.rst

@@ -51,6 +51,7 @@ Important notes:
 
 - When using -- to give options to py.test, you MUST also give borg.testsuite[.module].
 
+
 Building the docs with Sphinx
 -----------------------------
 
@@ -58,7 +59,7 @@ The documentation (in reStructuredText format, .rst) is in docs/.
 
 To build the html version of it, you need to have sphinx installed::
 
-  pip3 install sphinx
+  pip3 install sphinx  # important: this will install sphinx with Python 3
 
 Now run::
 
@@ -66,3 +67,73 @@ Now run::
   make html
 
 Then point a web browser at docs/_build/html/index.html.
+
+Using Vagrant
+-------------
+
+We use Vagrant for the automated creation of testing environment and borgbackup
+standalone binaries for various platforms.
+
+For better security, there is no automatic sync in the VM to host direction.
+The plugin `vagrant-scp` is useful to copy stuff from the VMs to the host.
+
+Usage::
+
+   To create and provision the VM:
+     vagrant up OS
+   To create an ssh session to the VM:
+     vagrant ssh OS command
+   To shut down the VM:
+     vagrant halt OS
+   To shut down and destroy the VM:
+     vagrant destroy OS
+   To copy files from the VM (in this case, the generated binary):
+     vagrant scp OS:/vagrant/borg/borg/dist/borg .
+
+
+Creating a new release
+----------------------
+
+Checklist::
+
+- all issues for this milestone closed?
+- any low hanging fruit left on the issue tracker?
+- run tox on all supported platforms via vagrant, check for test fails.
+- is Travis CI happy also?
+- update CHANGES.rst (compare to git log). check version number of upcoming release.
+- check MANIFEST.in and setup.py - are they complete?
+- tag the release::
+
+  git tag -s -m "tagged release" 0.26.0
+
+- cd docs ; make html  # to update the usage include files
+- update website with the html
+- create a release on PyPi::
+
+    python setup.py register sdist upload --identity="Thomas Waldmann" --sign
+
+- close release milestone.
+- announce on::
+
+  - mailing list
+  - Twitter
+  - IRC channel (topic)
+
+- create standalone binaries and link them from issue tracker: https://github.com/borgbackup/borg/issues/214
+
+
+Creating standalone binaries
+----------------------------
+
+Make sure you have everything built and installed (including llfuse and fuse).
+
+With virtual env activated::
+
+  pip install pyinstaller>=3.0  # or git checkout master
+  pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py
+  ls -l dist/*
+
+If you encounter issues, see also our `Vagrantfile` for details.
+
+Note: Standalone binaries built with pyinstaller are supposed to work on same OS,
+      same architecture (x86 32bit, amd64 64bit) without external dependencies.

docs/faq.rst

@@ -4,15 +4,9 @@
 Frequently asked questions
 ==========================
 
-Which platforms are supported?
-    Currently Linux, FreeBSD and MacOS X are supported.
-
-    You can try your luck on other POSIX-like systems, like Cygwin,
-    other BSDs, etc. but they are not officially supported.
-
 Can I backup VM disk images?
-    Yes, the :ref:`deduplication <deduplication_def>` technique used by |project_name|
-    makes sure only the modified parts of the file are stored.
+    Yes, the :ref:`deduplication <deduplication_def>` technique used by
+    |project_name| makes sure only the modified parts of the file are stored.
     Also, we have optional simple sparse file support for extract.
 
 Can I backup from multiple servers into a single repository?
@@ -41,14 +35,15 @@ Which file types, attributes, etc. are preserved?
     * User ID of owner
     * Group ID of owner
     * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
-    * Extended Attributes (xattrs)
+    * Extended Attributes (xattrs) on Linux, OS X and FreeBSD
     * Access Control Lists (ACL_) on Linux, OS X and FreeBSD
     * BSD flags on OS X and FreeBSD
 
 Which file types, attributes, etc. are *not* preserved?
-    * UNIX domain sockets (because it does not make sense - they are meaningless
-      without the running process that created them and the process needs to
-      recreate them in any case). So, don't panic if your backup misses a UDS!
+    * UNIX domain sockets (because it does not make sense - they are
+      meaningless without the running process that created them and the process
+      needs to recreate them in any case). So, don't panic if your backup
+      misses a UDS!
     * The precise on-disk representation of the holes in a sparse file.
       Archive creation has no special support for sparse files, holes are
       backed up as (deduplicated and compressed) runs of zero bytes.
@@ -76,52 +71,51 @@ When backing up to remote servers, do I have to trust the remote server?
     Yes, as an attacker with access to the remote server could delete (or
     otherwise make unavailable) all your backups.
 
-If a backup stops mid-way, does the already-backed-up data stay there? I.e. does |project_name| resume backups?
-    Yes, during a backup a special checkpoint archive named ``<archive-name>.checkpoint`` is saved every 5 minutes
-    containing all the data backed-up until that point. This means that at most 5 minutes worth of data needs to be
-    retransmitted if a backup needs to be restarted.
+If a backup stops mid-way, does the already-backed-up data stay there?
+    Yes, |project_name| supports resuming backups.
+    During a backup a special checkpoint archive named ``<archive-name>.checkpoint``
+    is saved every checkpoint interval (the default value for this is 5
+    minutes) containing all the data backed-up until that point. This means
+    that at most <checkpoint interval> worth of data needs to be retransmitted
+    if a backup needs to be restarted.
+    Once your backup has finished successfully, you can delete all ``*.checkpoint``
+    archives.
 
 If it crashes with a UnicodeError, what can I do?
     Check if your encoding is set correctly. For most POSIX-like systems, try::
 
         export LANG=en_US.UTF-8  # or similar, important is correct charset
 
-I can't extract non-ascii filenames by giving them on the commandline on OS X!?
-    This is due to different ways to represent some characters in unicode.
-    HFS+ likes the decomposed form while the commandline seems to be the composed
-    form usually. If you run into that, for now maybe just try:
+I can't extract non-ascii filenames by giving them on the commandline!?
+    This might be due to different ways to represent some characters in unicode
+    or due to other non-ascii encoding issues.
+    If you run into that, try this:
 
-    - avoiding the non-ascii characters on the commandline by e.g. extracting
+    - avoid the non-ascii characters on the commandline by e.g. extracting
       the parent directory (or even everything)
-    - try to enter the composed form on the commandline
     - mount the repo using FUSE and use some file manager
 
-    See issue #143 on the issue tracker for more about this.
-
-If I want to run |project_name| on a ARM CPU older than ARM v6?
-    You need to enable the alignment trap handler to fixup misaligned accesses::
-    
-        echo "2" > /proc/cpu/alignment
-
 Can |project_name| add redundancy to the backup data to deal with hardware malfunction?
-    No, it can't. While that at first sounds like a good idea to defend against some
-    defect HDD sectors or SSD flash blocks, dealing with this in a reliable way needs a lot
-    of low-level storage layout information and control which we do not have (and also can't
-    get, even if we wanted).
+    No, it can't. While that at first sounds like a good idea to defend against
+    some defect HDD sectors or SSD flash blocks, dealing with this in a
+    reliable way needs a lot of low-level storage layout information and
+    control which we do not have (and also can't get, even if we wanted).
+
+    So, if you need that, consider RAID or a filesystem that offers redundant
+    storage or just make backups to different locations / different hardware.
 
-    So, if you need that, consider RAID1 or a filesystem that offers redundant storage
-    or just make 2 backups to different locations / different hardware.
+    See also `ticket 225 <https://github.com/borgbackup/borg/issues/225>`_.
 
 Can |project_name| verify data integrity of a backup archive?
-    Yes, if you want to detect accidental data damage (like bit rot), use the ``check``
-    operation. It will notice corruption using CRCs and hashes.
-    If you want to be able to detect malicious tampering also, use a encrypted repo.
-    It will then be able to check using CRCs and HMACs.
+    Yes, if you want to detect accidental data damage (like bit rot), use the
+    ``check`` operation. It will notice corruption using CRCs and hashes.
+    If you want to be able to detect malicious tampering also, use a encrypted
+    repo. It will then be able to check using CRCs and HMACs.
 
 Why was Borg forked from Attic?
-    Borg was created in May 2015 in response to the difficulty of
-    getting new code or larger changes incorporated into Attic and
-    establishing a bigger developer community / more open development.
+    Borg was created in May 2015 in response to the difficulty of getting new
+    code or larger changes incorporated into Attic and establishing a bigger
+    developer community / more open development.
 
     More details can be found in `ticket 217
     <https://github.com/jborg/attic/issues/217>`_ that led to the fork.

docs/global.rst.inc

@@ -13,6 +13,7 @@
 .. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2
 .. _ACL: https://en.wikipedia.org/wiki/Access_control_list
 .. _libacl: http://savannah.nongnu.org/projects/acl/
+.. _libattr: http://savannah.nongnu.org/projects/attr/
 .. _liblz4: https://github.com/Cyan4973/lz4
 .. _OpenSSL: https://www.openssl.org/
 .. _Python: http://www.python.org/

docs/index.rst

@@ -16,3 +16,4 @@ Borg Documentation
    changes
    internals
    development
+   api

docs/installation.rst

@@ -4,11 +4,17 @@
 Installation
 ============
 
-|project_name| requires:
+|project_name| pyinstaller binary installation requires:
 
-* Python_ >= 3.2
+* Linux: glibc >= 2.12 (ok for most supported Linux releases)
+* MacOS X: 10.10 (unknown whether it works for older releases)
+* FreeBSD: 10.2 (unknown whether it works for older releases)
+
+|project_name| non-binary installation requires:
+
+* Python_ >= 3.2.2
 * OpenSSL_ >= 1.0.0
-* libacl_
+* libacl_ (that pulls in libattr_ also)
 * liblz4_
 * some python dependencies, see install_requires in setup.py
 
@@ -21,11 +27,10 @@ Below, we describe different ways to install |project_name|.
 
 - **dist package** - easy and fast, needs a distribution and platform specific
   binary package (for your Linux/*BSD/OS X/... distribution).
-- **wheel** - easy and fast, needs a platform specific borgbackup binary wheel,
-  which matches your platform [OS and CPU]).
+- **pyinstaller binary** - easy and fast, we provide a ready-to-use binary file
+  that just works on the supported platforms
 - **pypi** - installing a source package from pypi needs more installation steps
-  and will compile stuff - try this if there is no binary wheel that works for
-  you.
+  and will need a compiler, development headers, etc..
 - **git** - for developers and power users who want to have the latest code or
   use revision control (each release is tagged).
 
@@ -74,33 +79,13 @@ and compare that to our latest release and review the change log (see links on
 our web site).
 
 
-Debian Jessie / Ubuntu 14.04 preparations (wheel)
--------------------------------------------------
-
-.. parsed-literal::
-
-    # Python stuff we need
-    apt-get install python3 python3-pip
-
-    # Libraries we need (fuse is optional)
-    apt-get install openssl libacl1 liblz4-1 fuse
-
-
-Installation (wheel)
---------------------
-
-This uses the latest binary wheel release.
-
-.. parsed-literal::
-
-    # Check https://github.com/borgbackup/borg/issues/147 for the correct
-    # platform-specific binary wheel, download and install it:
+Installation (pyinstaller binary)
+---------------------------------
+For some platforms we offer a ready-to-use standalone borg binary.
 
-    # system-wide installation, needs sudo/root permissions:
-    sudo pip install borgbackup.whl
+It is supposed to work without requiring installation or preparations.
 
-    # home directory installation, no sudo/root needed:
-    pip install --user borgbackup.whl
+Check https://github.com/borgbackup/borg/issues/214 for available binaries.
 
 
 Debian Jessie / Ubuntu 14.04 preparations (git/pypi)
@@ -127,7 +112,7 @@ Debian Jessie / Ubuntu 14.04 preparations (git/pypi)
     # in case you get complaints about permission denied on /etc/fuse.conf:
     # on ubuntu this means your user is not in the "fuse" group. just add
     # yourself there, log out and log in again.
-    apt-get install libfuse-dev fuse
+    apt-get install libfuse-dev fuse pkg-config
 
     # optional: for unit testing
     apt-get install fakeroot
@@ -151,7 +136,7 @@ Korora / Fedora 21 preparations (git/pypi)
     sudo dnf install lz4-devel
 
     # optional: FUSE support - to mount backup archives
-    sudo dnf install fuse-devel fuse
+    sudo dnf install fuse-devel fuse pkgconfig
     
     # optional: for unit testing
     sudo dnf install fakeroot
@@ -201,7 +186,8 @@ This uses the latest (source package) release from PyPi.
     source borg-env/bin/activate   # always before using!
 
     # install borg + dependencies into virtualenv
-    pip install llfuse  # optional, for FUSE support
+    pip install 'llfuse<0.41'  # optional, for FUSE support
+                               # 0.41 and 0.41.1 have unicode issues at install time
     pip install borgbackup
 
 Note: we install into a virtual environment here, but this is not a requirement.
@@ -223,7 +209,8 @@ While we try not to break master, there are no guarantees on anything.
 
     # install borg + dependencies into virtualenv
     pip install sphinx  # optional, to build the docs
-    pip install llfuse  # optional, for FUSE support
+    pip install 'llfuse<0.41'  # optional, for FUSE support
+                               # 0.41 and 0.41.1 have unicode issues at install time
     cd borg
     pip install -r requirements.d/development.txt
     pip install -e .  # in-place editable mode

docs/quickstart.rst

@@ -85,9 +85,12 @@ certain number of old archives::
         --exclude /home/Ben/Music/Justin\ Bieber    \
         --exclude '*.pyc'
 
-    # Use the `prune` subcommand to maintain 7 daily, 4 weekly
-    # and 6 monthly archives.
-    borg prune -v $REPOSITORY --keep-daily=7 --keep-weekly=4 --keep-monthly=6
+    # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
+    # archives of THIS machine. --prefix `hostname`- is very important to
+    # limit prune's operation to this machine's archives and not apply to
+    # other machine's archives also.
+    borg prune -v $REPOSITORY --prefix `hostname`- \
+        --keep-daily=7 --keep-weekly=4 --keep-monthly=6
 
 .. backup_compression:
 

docs/update_usage.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-if [ ! -d usage ]; then
-  mkdir usage
-fi
-for cmd in change-passphrase check create delete extract info init list mount prune serve; do
-  FILENAME="usage/$cmd.rst.inc"
-  LINE=`echo -n borg $cmd | tr 'a-z- ' '-'`
-  echo -e ".. _borg_$cmd:\n" > $FILENAME
-  echo -e "borg $cmd\n$LINE\n::\n\n" >> $FILENAME
-  borg help $cmd --usage-only | sed -e 's/^/    /' >> $FILENAME
-  echo -e "\nDescription\n~~~~~~~~~~~\n" >> $FILENAME
-  borg help $cmd --epilog-only >> $FILENAME
-done

docs/usage.rst

@@ -48,6 +48,8 @@ General:
         can either leave it away or abbreviate as `::`, if a positional parameter is required.
     BORG_PASSPHRASE
         When set, use the value to answer the passphrase question for encrypted repositories.
+    BORG_RSH
+        When set, use this command instead of ``ssh``.
     TMPDIR
         where temporary files are stored (might need a lot of temporary space for some operations)
 
@@ -69,6 +71,8 @@ Directories:
 Building:
     BORG_OPENSSL_PREFIX
         Adds given OpenSSL header file directory to the default locations (setup.py).
+    BORG_LZ4_PREFIX
+        Adds given LZ4 header file directory to the default locations (setup.py).
 
 
 Please note:
@@ -212,12 +216,6 @@ Examples
     # Even slower, even higher compression (N = 0..9)
     $ borg create --compression lzma,N /mnt/backup::repo ~
 
-    # Backup some LV snapshots (you have to create the snapshots before this
-    # and remove them afterwards). We also backup the output of lvdisplay so
-    # we can see the LV sizes at restore time. See also "borg extract" examples.
-    $ lvdisplay > lvdisplay.txt
-    $ borg create --read-special /mnt/backup::repo lvdisplay.txt /dev/vg0/*-snapshot
-
 .. include:: usage/extract.rst.inc
 
 Examples
@@ -236,11 +234,6 @@ Examples
     # Extract the "src" directory but exclude object files
     $ borg extract /mnt/backup::my-files home/USERNAME/src --exclude '*.o'
 
-    # Restore LV snapshots (the target LVs /dev/vg0/* of correct size have
-    # to be already available and will be overwritten by this command!)
-    $ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root
-    $ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home
-
 Note: currently, extract always writes into the current working directory ("."),
       so make sure you ``cd`` to the right place before calling ``borg extract``.
 
@@ -274,10 +267,23 @@ Examples
 
 Examples
 ~~~~~~~~
+
+Be careful, prune is potentially dangerous command, it will remove backup
+archives.
+
+The default of prune is to apply to **all archives in the repository** unless
+you restrict its operation to a subset of the archives using `--prefix`.
+When using --prefix, be careful to choose a good prefix - e.g. do not use a
+prefix "foo" if you do not also want to match "foobar".
+
+It is strongly recommended to always run `prune --dry-run ...` first so you
+will see what it would do without it actually doing anything.
+
 ::
 
-    # Keep 7 end of day and 4 additional end of week archives:
-    $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4
+    # Keep 7 end of day and 4 additional end of week archives.
+    # Do a dry-run without actually deleting anything.
+    $ borg prune /mnt/backup --dry-run --keep-daily=7 --keep-weekly=4
 
     # Same as above but only apply to archive names starting with "foo":
     $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --prefix=foo
@@ -355,3 +361,70 @@ Examples
     $ cat ~/.ssh/authorized_keys
     command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...]
 
+
+Additional Notes
+================
+
+Here are misc. notes about topics that are maybe not covered in enough detail in the usage section.
+
+--read-special
+--------------
+
+The option --read-special is not intended for normal, filesystem-level (full or
+partly-recursive) backups. You only give this option if you want to do something
+rather ... special - and if you have hand-picked some files that you want to treat
+that way.
+
+`borg create --read-special` will open all files without doing any special treatment
+according to the file type (the only exception here are directories: they will be
+recursed into). Just imagine what happens if you do `cat filename` - the content
+you will see there is what borg will backup for that filename.
+
+So, for example, symlinks will be followed, block device content will be read,
+named pipes / UNIX domain sockets will be read.
+
+You need to be careful with what you give as filename when using --read-special,
+e.g. if you give /dev/zero, your backup will never terminate.
+
+The given files' metadata is saved as it would be saved without --read-special
+(e.g. its name, its size [might be 0], its mode, etc.) - but additionally, also
+the content read from it will be saved for it.
+
+Restoring such files' content is currently only supported one at a time via --stdout
+option (and you have to redirect stdout to where ever it shall go, maybe directly
+into an existing device file of your choice or indirectly via dd).
+
+Example
+~~~~~~~
+
+Imagine you have made some snapshots of logical volumes (LVs) you want to backup.
+
+Note: For some scenarios, this is a good method to get "crash-like" consistency
+(I call it crash-like because it is the same as you would get if you just hit the
+reset button or your machine would abrubtly and completely crash).
+This is better than no consistency at all and a good method for some use cases,
+but likely not good enough if you have databases running.
+
+Then you create a backup archive of all these snapshots. The backup process will
+see a "frozen" state of the logical volumes, while the processes working in the
+original volumes continue changing the data stored there.
+
+You also add the output of `lvdisplay` to your backup, so you can see the LV sizes
+in case you ever need to recreate and restore them.
+
+After the backup has completed, you remove the snapshots again.
+
+::
+    $ # create snapshots here
+    $ lvdisplay > lvdisplay.txt
+    $ borg create --read-special /mnt/backup::repo lvdisplay.txt /dev/vg0/*-snapshot
+    $ # remove snapshots here
+
+Now, let's see how to restore some LVs from such a backup.
+
+    $ borg extract /mnt/backup::repo lvdisplay.txt
+    $ # create empty LVs with correct sizes here (look into lvdisplay.txt).
+    $ # we assume that you created an empty root and home LV and overwrite it now:
+    $ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root
+    $ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home
+

setup.cfg

@@ -4,5 +4,5 @@ python_files = testsuite/*.py
 [flake8]
 ignore = E226,F403
 max-line-length = 250
-exclude = versioneer.py,docs/conf.py,borg/_version.py,build,dist,.git,.idea,.cache
+exclude = docs/conf.py,borg/_version.py,build,dist,.git,.idea,.cache
 max-complexity = 100

setup.py

@@ -4,10 +4,16 @@ import sys
 from glob import glob
 
 min_python = (3, 2)
-if sys.version_info < min_python:
+my_python = sys.version_info
+
+if my_python < min_python:
     print("Borg requires Python %d.%d or later" % min_python)
     sys.exit(1)
 
+# msgpack pure python data corruption was fixed in 0.4.6.
+# Also, we might use some rather recent API features.
+install_requires=['msgpack-python>=0.4.6', ]
+
 
 from setuptools import setup, Extension
 from setuptools.command.sdist import sdist
@@ -59,7 +65,7 @@ except ImportError:
     if not all(os.path.exists(path) for path in [
         compress_source, crypto_source, chunker_source, hashindex_source,
         platform_linux_source, platform_freebsd_source]):
-        raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version')
+        raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 
 def detect_openssl(prefixes):
@@ -71,14 +77,36 @@ def detect_openssl(prefixes):
                     return prefix
 
 
+def detect_lz4(prefixes):
+    for prefix in prefixes:
+        filename = os.path.join(prefix, 'include', 'lz4.h')
+        if os.path.exists(filename):
+            with open(filename, 'r') as fd:
+                if 'LZ4_decompress_safe' in fd.read():
+                    return prefix
+
+
+include_dirs = []
+library_dirs = []
+
 possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local']
 if os.environ.get('BORG_OPENSSL_PREFIX'):
     possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
 ssl_prefix = detect_openssl(possible_openssl_prefixes)
 if not ssl_prefix:
     raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
-include_dirs = [os.path.join(ssl_prefix, 'include')]
-library_dirs = [os.path.join(ssl_prefix, 'lib')]
+include_dirs.append(os.path.join(ssl_prefix, 'include'))
+library_dirs.append(os.path.join(ssl_prefix, 'lib'))
+
+
+possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local']
+if os.environ.get('BORG_LZ4_PREFIX'):
+    possible_openssl_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
+lz4_prefix = detect_lz4(possible_lz4_prefixes)
+if not lz4_prefix:
+    raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
+include_dirs.append(os.path.join(lz4_prefix, 'include'))
+library_dirs.append(os.path.join(lz4_prefix, 'lib'))
 
 
 with open('README.rst', 'r') as fd:
@@ -87,7 +115,7 @@ with open('README.rst', 'r') as fd:
 cmdclass = {'build_ext': build_ext, 'sdist': Sdist}
 
 ext_modules = [
-    Extension('borg.compress', [compress_source], libraries=['lz4']),
+    Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs),
     Extension('borg.chunker', [chunker_source]),
     Extension('borg.hashindex', [hashindex_source])
@@ -110,13 +138,15 @@ setup(
     description='Deduplicated, encrypted, authenticated and compressed backups',
     long_description=long_description,
     license='BSD',
-    platforms=['Linux', 'MacOS X', 'FreeBSD', ],
+    platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
     classifiers=[
         'Development Status :: 4 - Beta',
         'Environment :: Console',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: BSD License',
         'Operating System :: POSIX :: BSD :: FreeBSD',
+        'Operating System :: POSIX :: BSD :: OpenBSD',
+        'Operating System :: POSIX :: BSD :: NetBSD',
         'Operating System :: MacOS :: MacOS X',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python',
@@ -124,10 +154,11 @@ setup(
         'Programming Language :: Python :: 3.2',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Topic :: Security :: Cryptography',
         'Topic :: System :: Archiving :: Backup',
     ],
-    packages=['borg', 'borg.testsuite'],
+    packages=['borg', 'borg.testsuite', 'borg.support', ],
     entry_points={
         'console_scripts': [
             'borg = borg.archiver:main',
@@ -136,7 +167,5 @@ setup(
     cmdclass=cmdclass,
     ext_modules=ext_modules,
     setup_requires=['setuptools_scm>=1.7'],
-    # msgpack pure python data corruption was fixed in 0.4.6.
-    # Also, we might use some rather recent API features.
-    install_requires=['msgpack-python>=0.4.6'],
+    install_requires=install_requires,
 )

tox.ini

@@ -2,13 +2,15 @@
 # fakeroot -u tox --recreate
 
 [tox]
-envlist = py32, py33, py34
+envlist = py{32,33,34,35}
 
 [testenv]
 # Change dir to avoid import problem for cython code. The directory does
 # not really matter, should be just different from the toplevel dir.
 changedir = {toxworkdir}
-deps = -rrequirements.d/development.txt
+deps =
+     -rrequirements.d/development.txt
+     attic
 commands = py.test --cov=borg --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *