Merge pull request #408 from anarcat/static-usage

Static usage generation

.gitignore

@@ -15,8 +15,6 @@ platform_linux.c
 *.pyc
 *.pyo
 *.so
-docs/usage/*.inc
-docs/api.rst
 .idea/
 .cache/
 borg/_version.py

borg/archive.py

@@ -18,16 +18,12 @@ import time
 from io import BytesIO
 from . import xattr
 from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \
-    Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython, \
+    Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \
     st_atime_ns, st_ctime_ns, st_mtime_ns
-if have_cython():
-    from .platform import acl_get, acl_set
-    from .chunker import Chunker
-    from .hashindex import ChunkIndex
-    import msgpack
-else:
-    import mock
-    msgpack = mock.Mock()
+from .platform import acl_get, acl_set
+from .chunker import Chunker
+from .hashindex import ChunkIndex
+import msgpack
 
 ITEMS_BUFFER = 1024 * 1024
 

borg/archiver.py

@@ -20,16 +20,15 @@ from .helpers import Error, location_validator, format_time, format_file_size, \
     format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
     get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \
     Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
-    dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \
+    dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .logger import create_logger, setup_logging
 logger = create_logger()
-if have_cython():
-    from .compress import Compressor, COMPR_BUFFER
-    from .upgrader import AtticRepositoryUpgrader
-    from .repository import Repository
-    from .cache import Cache
-    from .key import key_creator
+from .compress import Compressor, COMPR_BUFFER
+from .upgrader import AtticRepositoryUpgrader
+from .repository import Repository
+from .cache import Cache
+from .key import key_creator
 from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
 from .remote import RepositoryServer, RemoteRepository
 

borg/cache.py

@@ -10,12 +10,11 @@ from .key import PlaintextKey
 from .logger import create_logger
 logger = create_logger()
 from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \
-    bigint_to_int, format_file_size, have_cython, yes
+    bigint_to_int, format_file_size, yes
 from .locking import UpgradableLock
 from .hashindex import ChunkIndex
 
-if have_cython():
-    import msgpack
+import msgpack
 
 
 class Cache:

borg/fuse.py

@@ -7,11 +7,10 @@ import stat
 import tempfile
 import time
 from .archive import Archive
-from .helpers import daemonize, have_cython
+from .helpers import daemonize
 from .remote import cache_if_remote
 
-if have_cython():
-    import msgpack
+import msgpack
 
 # Does this version of llfuse support ns precision?
 have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')

borg/helpers.py

@@ -22,27 +22,11 @@ from fnmatch import translate
 from operator import attrgetter
 
 
-def have_cython():
-    """allow for a way to disable Cython includes
-
-    this is used during usage docs build, in setup.py. It is to avoid
-    loading the Cython libraries which are built, but sometimes not in
-    the search path (namely, during Tox runs).
-
-    we simply check an environment variable (``BORG_CYTHON_DISABLE``)
-    which, when set (to anything) will disable includes of Cython
-    libraries in key places to enable usage docs to be built.
-
-    :returns: True if Cython is available, False otherwise.
-    """
-    return not os.environ.get('BORG_CYTHON_DISABLE')
-
-if have_cython():
-    from . import hashindex
-    from . import chunker
-    from . import crypto
-    import msgpack
-    import msgpack.fallback
+from . import hashindex
+from . import chunker
+from . import crypto
+import msgpack
+import msgpack.fallback
 
 
 # return codes returned by borg command

borg/key.py

@@ -7,14 +7,13 @@ import textwrap
 import hmac
 from hashlib import sha256
 
-from .helpers import IntegrityError, get_keys_dir, Error, have_cython
+from .helpers import IntegrityError, get_keys_dir, Error
 from .logger import create_logger
 logger = create_logger()
 
-if have_cython():
-    from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
-    from .compress import Compressor, COMPR_BUFFER
-    import msgpack
+from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
+from .compress import Compressor, COMPR_BUFFER
+import msgpack
 
 PREFIX = b'\0' * 8
 

borg/remote.py

@@ -10,11 +10,10 @@ import traceback
 
 from . import __version__
 
-from .helpers import Error, IntegrityError, have_cython
+from .helpers import Error, IntegrityError
 from .repository import Repository
 
-if have_cython():
-    import msgpack
+import msgpack
 
 BUFSIZE = 10 * 1024 * 1024
 

borg/repository.py

@@ -10,9 +10,8 @@ import shutil
 import struct
 from zlib import crc32
 
-from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython
-if have_cython():
-    from .hashindex import NSIndex
+from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify
+from .hashindex import NSIndex
 from .locking import UpgradableLock
 from .lrucache import LRUCache
 

docs/api.rst

@@ -0,0 +1,87 @@
+
+API Documentation
+=================
+
+.. automodule:: borg.key
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.cache
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.locking
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.platform
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.xattr
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.fuse
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.logger
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.repository
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.archiver
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.archive
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.lrucache
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.remote
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.upgrader
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.helpers
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.platform_freebsd
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.hashindex
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.chunker
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.platform_darwin
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.platform_linux
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.compress
+    :members:
+    :undoc-members:
+
+.. automodule:: borg.crypto
+    :members:
+    :undoc-members:

docs/development.rst

@@ -61,6 +61,23 @@ Important notes:
 - When using -- to give options to py.test, you MUST also give borg.testsuite[.module].
 
 
+Regenerate usage files
+----------------------
+
+Usage and API documentation is currently committed directly to git,
+although those files are generated automatically from the source
+tree.
+
+When a new module is added, the ``docs/api.rst`` file needs to be
+regenerated::
+
+  ./setup.py build_api
+
+When a command is added, a commandline flag changed, added or removed,
+the usage docs need to be rebuilt as well::
+
+  ./setup.py build_usage
+
 Building the docs with Sphinx
 -----------------------------
 

docs/usage.rst

@@ -59,12 +59,6 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do
         For "Warning: The repository at location ... was previously located at ..."
     BORG_CHECK_I_KNOW_WHAT_I_AM_DOING
         For "Warning: 'check --repair' is an experimental feature that might result in data loss."
-    BORG_CYTHON_DISABLE
-        Disables the loading of Cython modules. This is currently
-        experimental and is used only to generate usage docs at build
-        time. It is unlikely to produce good results on a regular
-        run. The variable should be set to the name of the  calling class, and
-        should be unique across all of borg. It is currently only used by ``build_usage``.
 
 Directories:
     BORG_KEYS_DIR

docs/usage/change-passphrase.rst.inc

@@ -0,0 +1,29 @@
+.. _borg_change-passphrase:
+
+borg change-passphrase
+----------------------
+::
+
+    usage: borg change-passphrase [-h] [-v] [--show-rc] [--no-files-cache]
+                                  [--umask M] [--remote-path PATH]
+                                  [REPOSITORY]
+    
+    Change repository key file passphrase
+    
+    positional arguments:
+      REPOSITORY
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+The key files used for repository encryption are optionally passphrase
+protected. This command can be used to change this passphrase.

docs/usage/check.rst.inc

@@ -0,0 +1,67 @@
+.. _borg_check:
+
+borg check
+----------
+::
+
+    usage: borg check [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                      [--remote-path PATH] [--repository-only] [--archives-only]
+                      [--repair] [--last N]
+                      [REPOSITORY_OR_ARCHIVE]
+    
+    Check repository consistency
+    
+    positional arguments:
+      REPOSITORY_OR_ARCHIVE
+                            repository or archive to check consistency of
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      --repository-only     only perform repository checks
+      --archives-only       only perform archives checks
+      --repair              attempt to repair any inconsistencies found
+      --last N              only check last N archives (Default: all)
+    
+Description
+~~~~~~~~~~~
+
+The check command verifies the consistency of a repository and the corresponding archives.
+
+First, the underlying repository data files are checked:
+
+- For all segments the segment magic (header) is checked
+- For all objects stored in the segments, all metadata (e.g. crc and size) and
+  all data is read. The read data is checked by size and CRC. Bit rot and other
+  types of accidental damage can be detected this way.
+- If we are in repair mode and a integrity error is detected for a segment,
+  we try to recover as many objects from the segment as possible.
+- In repair mode, it makes sure that the index is consistent with the data
+  stored in the segments.
+- If you use a remote repo server via ssh:, the repo check is executed on the
+  repo server without causing significant network traffic.
+- The repository check can be skipped using the --archives-only option.
+
+Second, the consistency and correctness of the archive metadata is verified:
+
+- Is the repo manifest present? If not, it is rebuilt from archive metadata
+  chunks (this requires reading and decrypting of all metadata and data).
+- Check if archive metadata chunk is present. if not, remove archive from
+  manifest.
+- For all files (items) in the archive, for all chunks referenced by these
+  files, check if chunk is present (if not and we are in repair mode, replace
+  it with a same-size chunk of zeros). This requires reading of archive and
+  file metadata, but not data.
+- If we are in repair mode and we checked all the archives: delete orphaned
+  chunks from the repo.
+- if you use a remote repo server via ssh:, the archive check is executed on
+  the client machine (because if encryption is enabled, the checks will require
+  decryption and this is always done client-side, because key access will be
+  required).
+- The archive checks can be time consuming, they can be skipped using the
+  --repository-only option.

docs/usage/create.rst.inc

@@ -0,0 +1,73 @@
+.. _borg_create:
+
+borg create
+-----------
+::
+
+    usage: borg create [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                       [--remote-path PATH] [-s] [-p] [-e PATTERN]
+                       [--exclude-from EXCLUDEFILE] [--exclude-caches]
+                       [--exclude-if-present FILENAME] [--keep-tag-files]
+                       [-c SECONDS] [-x] [--numeric-owner]
+                       [--timestamp yyyy-mm-ddThh:mm:ss]
+                       [--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE]
+                       [-C COMPRESSION] [--read-special] [-n]
+                       ARCHIVE PATH [PATH ...]
+    
+    Create new archive
+    
+    positional arguments:
+      ARCHIVE               name of archive to create (must be also a valid
+                            directory name)
+      PATH                  paths to archive
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      -s, --stats           print statistics for the created archive
+      -p, --progress        toggle progress display while creating the archive,
+                            showing Original, Compressed and Deduplicated sizes,
+                            followed by the Number of files seen and the path
+                            being processed, default: True
+      -e PATTERN, --exclude PATTERN
+                            exclude paths matching PATTERN
+      --exclude-from EXCLUDEFILE
+                            read exclude patterns from EXCLUDEFILE, one per line
+      --exclude-caches      exclude directories that contain a CACHEDIR.TAG file
+                            (http://www.brynosaurus.com/cachedir/spec.html)
+      --exclude-if-present FILENAME
+                            exclude directories that contain the specified file
+      --keep-tag-files      keep tag files of excluded caches/directories
+      -c SECONDS, --checkpoint-interval SECONDS
+                            write checkpoint every SECONDS seconds (Default: 300)
+      -x, --one-file-system
+                            stay in same file system, do not cross mount points
+      --numeric-owner       only store numeric user and group identifiers
+      --timestamp yyyy-mm-ddThh:mm:ss
+                            manually specify the archive creation date/time (UTC).
+                            alternatively, give a reference file/directory.
+      --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE
+                            specify the chunker parameters. default: 10,23,16,4095
+      -C COMPRESSION, --compression COMPRESSION
+                            select compression algorithm (and level): none == no
+                            compression (default), lz4 == lz4, zlib == zlib
+                            (default level 6), zlib,0 .. zlib,9 == zlib (with
+                            level 0..9), lzma == lzma (default level 6), lzma,0 ..
+                            lzma,9 == lzma (with level 0..9).
+      --read-special        open and read special files as if they were regular
+                            files
+      -n, --dry-run         do not create a backup archive
+    
+Description
+~~~~~~~~~~~
+
+This command creates a backup archive containing all files found while recursively
+traversing all paths specified. The archive will consume almost no disk space for
+files or parts of files that have already been stored in other archives.
+
+See the output of the "borg help patterns" command for more help on exclude patterns.

docs/usage/debug-delete-obj.rst.inc

@@ -0,0 +1,29 @@
+.. _borg_debug-delete-obj:
+
+borg debug-delete-obj
+---------------------
+::
+
+    usage: borg debug-delete-obj [-h] [-v] [--show-rc] [--no-files-cache]
+                                 [--umask M] [--remote-path PATH]
+                                 [REPOSITORY] IDs [IDs ...]
+    
+    delete the objects with the given IDs from the repo
+    
+    positional arguments:
+      REPOSITORY          repository to use
+      IDs                 hex object ID(s) to delete from the repo
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command deletes objects from the repository.

docs/usage/debug-dump-archive-items.rst.inc

@@ -0,0 +1,28 @@
+.. _borg_debug-dump-archive-items:
+
+borg debug-dump-archive-items
+-----------------------------
+::
+
+    usage: borg debug-dump-archive-items [-h] [-v] [--show-rc] [--no-files-cache]
+                                         [--umask M] [--remote-path PATH]
+                                         ARCHIVE
+    
+    dump (decrypted, decompressed) archive items metadata (not: data)
+    
+    positional arguments:
+      ARCHIVE             archive to dump
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.

docs/usage/debug-get-obj.rst.inc

@@ -0,0 +1,30 @@
+.. _borg_debug-get-obj:
+
+borg debug-get-obj
+------------------
+::
+
+    usage: borg debug-get-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                              [--remote-path PATH]
+                              [REPOSITORY] ID PATH
+    
+    get object contents from the repository and write it into file
+    
+    positional arguments:
+      REPOSITORY          repository to use
+      ID                  hex object ID to get from the repo
+      PATH                file to write object data into
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command gets an object from the repository.

docs/usage/debug-put-obj.rst.inc

@@ -0,0 +1,29 @@
+.. _borg_debug-put-obj:
+
+borg debug-put-obj
+------------------
+::
+
+    usage: borg debug-put-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                              [--remote-path PATH]
+                              [REPOSITORY] PATH [PATH ...]
+    
+    put file(s) contents into the repository
+    
+    positional arguments:
+      REPOSITORY          repository to use
+      PATH                file(s) to read and create object(s) from
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command puts objects into the repository.

docs/usage/delete.rst.inc

@@ -0,0 +1,32 @@
+.. _borg_delete:
+
+borg delete
+-----------
+::
+
+    usage: borg delete [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                       [--remote-path PATH] [-s] [-c]
+                       [TARGET]
+    
+    Delete an existing repository or archive
+    
+    positional arguments:
+      TARGET              archive or repository to delete
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+      -s, --stats         print statistics for the deleted archive
+      -c, --cache-only    delete only the local cache for the given repository
+    
+Description
+~~~~~~~~~~~
+
+This command deletes an archive from the repository or the complete repository.
+Disk space is reclaimed accordingly. If you delete the complete repository, the
+local cache for it (if any) is also deleted.

docs/usage/extract.rst.inc

@@ -0,0 +1,49 @@
+.. _borg_extract:
+
+borg extract
+------------
+::
+
+    usage: borg extract [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                        [--remote-path PATH] [-n] [-e PATTERN]
+                        [--exclude-from EXCLUDEFILE] [--numeric-owner]
+                        [--strip-components NUMBER] [--stdout] [--sparse]
+                        ARCHIVE [PATH [PATH ...]]
+    
+    Extract archive contents
+    
+    positional arguments:
+      ARCHIVE               archive to extract
+      PATH                  paths to extract
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      -n, --dry-run         do not actually change any files
+      -e PATTERN, --exclude PATTERN
+                            exclude paths matching PATTERN
+      --exclude-from EXCLUDEFILE
+                            read exclude patterns from EXCLUDEFILE, one per line
+      --numeric-owner       only obey numeric user and group identifiers
+      --strip-components NUMBER
+                            Remove the specified number of leading path elements.
+                            Pathnames with fewer elements will be silently
+                            skipped.
+      --stdout              write all extracted data to stdout
+      --sparse              create holes in output sparse file from all-zero
+                            chunks
+    
+Description
+~~~~~~~~~~~
+
+This command extracts the contents of an archive. By default the entire
+archive is extracted but a subset of files and directories can be selected
+by passing a list of ``PATHs`` as arguments. The file selection can further
+be restricted by using the ``--exclude`` option.
+
+See the output of the "borg help patterns" command for more help on exclude patterns.

docs/usage/help.rst.inc

@@ -0,0 +1,34 @@
+.. _borg_patterns:
+
+borg help patterns
+~~~~~~~~~~~~~~~~~~
+::
+
+
+        Exclude patterns use a variant of shell pattern syntax, with '*' matching any
+        number of characters, '?' matching any single character, '[...]' matching any
+        single character specified, including ranges, and '[!...]' matching any
+        character not specified.  For the purpose of these patterns, the path
+        separator ('\' for Windows and '/' on other systems) is not treated
+        specially.  For a path to match a pattern, it must completely match from
+        start to end, or must match from the start to just before a path separator.
+        Except for the root path, paths will never end in the path separator when
+        matching is attempted.  Thus, if a given pattern ends in a path separator, a
+        '*' is appended before matching is attempted.  Patterns with wildcards should
+        be quoted to protect them from shell expansion.
+
+        Examples:
+
+        # Exclude '/home/user/file.o' but not '/home/user/file.odt':
+        $ borg create -e '*.o' backup /
+
+        # Exclude '/home/user/junk' and '/home/user/subdir/junk' but
+        # not '/home/user/importantjunk' or '/etc/junk':
+        $ borg create -e '/home/*/junk' backup /
+
+        # Exclude the contents of '/home/user/cache' but not the directory itself:
+        $ borg create -e /home/user/cache/ backup /
+
+        # The file '/home/user/cache/important' is *not* backed up:
+        $ borg create -e /home/user/cache/ backup / /home/user/cache/important
+        

docs/usage/info.rst.inc

@@ -0,0 +1,28 @@
+.. _borg_info:
+
+borg info
+---------
+::
+
+    usage: borg info [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                     [--remote-path PATH]
+                     ARCHIVE
+    
+    Show archive details such as disk space used
+    
+    positional arguments:
+      ARCHIVE             archive to display information about
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command displays some detailed information about the specified archive.

docs/usage/init.rst.inc

@@ -0,0 +1,34 @@
+.. _borg_init:
+
+borg init
+---------
+::
+
+    usage: borg init [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                     [--remote-path PATH] [-e {none,keyfile,repokey,passphrase}]
+                     [REPOSITORY]
+    
+    Initialize an empty repository
+    
+    positional arguments:
+      REPOSITORY            repository to create
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      -e {none,keyfile,repokey,passphrase}, --encryption {none,keyfile,repokey,passphrase}
+                            select encryption key mode
+    
+Description
+~~~~~~~~~~~
+
+This command initializes an empty repository. A repository is a filesystem
+directory containing the deduplicated data from zero or more archives.
+Encryption can be enabled at repository init time.
+Please note that the 'passphrase' encryption mode is DEPRECATED (instead of it,
+consider using 'repokey').

docs/usage/list.rst.inc

@@ -0,0 +1,32 @@
+.. _borg_list:
+
+borg list
+---------
+::
+
+    usage: borg list [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                     [--remote-path PATH] [--short] [-p PREFIX]
+                     [REPOSITORY_OR_ARCHIVE]
+    
+    List archive or repository contents
+    
+    positional arguments:
+      REPOSITORY_OR_ARCHIVE
+                            repository/archive to list contents of
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      --short               only print file/directory names, nothing else
+      -p PREFIX, --prefix PREFIX
+                            only consider archive names starting with this prefix
+    
+Description
+~~~~~~~~~~~
+
+This command lists the contents of a repository or an archive.

docs/usage/mount.rst.inc

@@ -0,0 +1,35 @@
+.. _borg_mount:
+
+borg mount
+----------
+::
+
+    usage: borg mount [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                      [--remote-path PATH] [-f] [-o OPTIONS]
+                      REPOSITORY_OR_ARCHIVE MOUNTPOINT
+    
+    Mount archive or an entire repository as a FUSE fileystem
+    
+    positional arguments:
+      REPOSITORY_OR_ARCHIVE
+                            repository/archive to mount
+      MOUNTPOINT            where to mount filesystem
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      -f, --foreground      stay in foreground, do not daemonize
+      -o OPTIONS            Extra mount options
+    
+Description
+~~~~~~~~~~~
+
+This command mounts an archive as a FUSE filesystem. This can be useful for
+browsing an archive or restoring individual files. Unless the ``--foreground``
+option is given the command will run in the background until the filesystem
+is ``umounted``.

docs/usage/prune.rst.inc

@@ -0,0 +1,66 @@
+.. _borg_prune:
+
+borg prune
+----------
+::
+
+    usage: borg prune [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                      [--remote-path PATH] [-n] [-s] [--keep-within WITHIN]
+                      [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY]
+                      [-p PREFIX]
+                      [REPOSITORY]
+    
+    Prune repository archives according to specified rules
+    
+    positional arguments:
+      REPOSITORY            repository to prune
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      -n, --dry-run         do not change repository
+      -s, --stats           print statistics for the deleted archive
+      --keep-within WITHIN  keep all archives within this time interval
+      -H HOURLY, --keep-hourly HOURLY
+                            number of hourly archives to keep
+      -d DAILY, --keep-daily DAILY
+                            number of daily archives to keep
+      -w WEEKLY, --keep-weekly WEEKLY
+                            number of weekly archives to keep
+      -m MONTHLY, --keep-monthly MONTHLY
+                            number of monthly archives to keep
+      -y YEARLY, --keep-yearly YEARLY
+                            number of yearly archives to keep
+      -p PREFIX, --prefix PREFIX
+                            only consider archive names starting with this prefix
+    
+Description
+~~~~~~~~~~~
+
+The prune command prunes a repository by deleting archives not matching
+any of the specified retention options. This command is normally used by
+automated backup scripts wanting to keep a certain number of historic backups.
+
+As an example, "-d 7" means to keep the latest backup on each day for 7 days.
+Days without backups do not count towards the total.
+The rules are applied from hourly to yearly, and backups selected by previous
+rules do not count towards those of later rules. The time that each backup
+completes is used for pruning purposes. Dates and times are interpreted in
+the local timezone, and weeks go from Monday to Sunday. Specifying a
+negative number of archives to keep means that there is no limit.
+
+The "--keep-within" option takes an argument of the form "<int><char>",
+where char is "H", "d", "w", "m", "y". For example, "--keep-within 2d" means
+to keep all archives that were created within the past 48 hours.
+"1m" is taken to mean "31d". The archives kept with this option do not
+count towards the totals specified by any other options.
+
+If a prefix is set with -p, then only archives that start with the prefix are
+considered for deletion and only those archives count towards the totals
+specified by the rules.
+Otherwise, *all* archives in the repository are candidates for deletion!

docs/usage/rename.rst.inc

@@ -0,0 +1,29 @@
+.. _borg_rename:
+
+borg rename
+-----------
+::
+
+    usage: borg rename [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                       [--remote-path PATH]
+                       ARCHIVE NEWNAME
+    
+    Rename an existing archive
+    
+    positional arguments:
+      ARCHIVE             archive to rename
+      NEWNAME             the new archive name to use
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+    
+Description
+~~~~~~~~~~~
+
+This command renames an archive in the repository.

docs/usage/serve.rst.inc

@@ -0,0 +1,27 @@
+.. _borg_serve:
+
+borg serve
+----------
+::
+
+    usage: borg serve [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                      [--remote-path PATH] [--restrict-to-path PATH]
+    
+    Start in server mode. This command is usually not used manually.
+            
+    
+    optional arguments:
+      -h, --help            show this help message and exit
+      -v, --verbose         verbose output
+      --show-rc             show/log the return code (rc)
+      --no-files-cache      do not load/update the file metadata cache used to
+                            detect unchanged files
+      --umask M             set umask to M (local and remote, default: 63)
+      --remote-path PATH    set remote path to executable (default: "borg")
+      --restrict-to-path PATH
+                            restrict repository access to PATH
+    
+Description
+~~~~~~~~~~~
+
+This command starts a repository server process. This command is usually not used manually.

docs/usage/upgrade.rst.inc

@@ -0,0 +1,65 @@
+.. _borg_upgrade:
+
+borg upgrade
+------------
+::
+
+    usage: borg upgrade [-h] [-v] [--show-rc] [--no-files-cache] [--umask M]
+                        [--remote-path PATH] [-n] [-i]
+                        [REPOSITORY]
+    
+    upgrade a repository from a previous version
+    
+    positional arguments:
+      REPOSITORY          path to the repository to be upgraded
+    
+    optional arguments:
+      -h, --help          show this help message and exit
+      -v, --verbose       verbose output
+      --show-rc           show/log the return code (rc)
+      --no-files-cache    do not load/update the file metadata cache used to
+                          detect unchanged files
+      --umask M           set umask to M (local and remote, default: 63)
+      --remote-path PATH  set remote path to executable (default: "borg")
+      -n, --dry-run       do not change repository
+      -i, --inplace       rewrite repository in place, with no chance of going
+                          back to older versions of the repository.
+    
+Description
+~~~~~~~~~~~
+
+upgrade an existing Borg repository. this currently
+only support converting an Attic repository, but may
+eventually be extended to cover major Borg upgrades as well.
+
+it will change the magic strings in the repository's segments
+to match the new Borg magic strings. the keyfiles found in
+$ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and
+copied to $BORG_KEYS_DIR or ~/.borg/keys.
+
+the cache files are converted, from $ATTIC_CACHE_DIR or
+~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the
+cache layout between Borg and Attic changed, so it is possible
+the first backup after the conversion takes longer than expected
+due to the cache resync.
+
+upgrade should be able to resume if interrupted, although it
+will still iterate over all segments. if you want to start
+from scratch, use `borg delete` over the copied repository to
+make sure the cache files are also removed:
+
+    borg delete borg
+
+unless ``--inplace`` is specified, the upgrade process first
+creates a backup copy of the repository, in
+REPOSITORY.upgrade-DATETIME, using hardlinks. this takes
+longer than in place upgrades, but is much safer and gives
+progress information (as opposed to ``cp -al``). once you are
+satisfied with the conversion, you can safely destroy the
+backup copy.
+
+WARNING: running the upgrade in place will make the current
+copy unusable with older version, with no way of going back
+to previous versions. this can PERMANENTLY DAMAGE YOUR
+REPOSITORY!  Attic CAN NOT READ BORG REPOSITORIES, as the
+magic strings have changed. you have been warned.

setup.py

@@ -6,8 +6,6 @@ from glob import glob
 
 from distutils.command.build import build
 from distutils.core import Command
-from distutils import log
-from setuptools.command.build_py import build_py
 
 min_python = (3, 2)
 my_python = sys.version_info
@@ -137,8 +135,6 @@ class build_usage(Command):
     def run(self):
         print('generating usage docs')
         # allows us to build docs without the C modules fully loaded during help generation
-        if 'BORG_CYTHON_DISABLE' not in os.environ:
-            os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__
         from borg.archiver import Archiver
         parser = Archiver().build_parser(prog='borg')
         choices = {}
@@ -168,9 +164,6 @@ class build_usage(Command):
                     doc.write(re.sub("^", "    ", parser.format_help(), flags=re.M))
                     doc.write("\nDescription\n~~~~~~~~~~~\n")
                     doc.write(epilog)
-        # return to regular Cython configuration, if we changed it
-        if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__:
-            del os.environ['BORG_CYTHON_DISABLE']
 
 
 class build_api(Command):
@@ -203,39 +196,10 @@ API Documentation
 """ % mod)
 
 
-class build_py_custom(build_py):
-    """override build_py to also build our stuff
-
-    it is unclear why this is necessary, but in some environments
-    (Readthedocs.org, specifically), the above
-    ``build.sub_commands.append()`` doesn't seem to have an effect:
-    our custom build commands seem to be ignored when running
-    ``setup.py install``.
-
-    This class overrides the ``build_py`` target by forcing it to run
-    our custom steps as well.
-
-    See also the `bug report on RTD
-    <https://github.com/rtfd/readthedocs.org/issues/1740>`_.
-    """
-    def run(self):
-        super().run()
-        self.announce('calling custom build steps', level=log.INFO)
-        self.run_command('build_ext')
-        if on_rtd:
-            # only build these files if running on readthedocs, but not
-            # for a normal production install. It requires "mock" and we
-            # do not have that as a build dependency. Also, for really
-            # building the docs, it would also require sphinx, etc.
-            self.run_command('build_api')
-            self.run_command('build_usage')
-
-
 cmdclass = {
     'build_ext': build_ext,
     'build_api': build_api,
     'build_usage': build_usage,
-    'build_py': build_py_custom,
     'sdist': Sdist
 }