Browse Source

Merge pull request #6793 from ThomasWaldmann/more-fixes-borg2

more fixes/updates for borg2
TW 2 years ago
parent
commit
b157573c69

+ 0 - 24
AUTHORS

@@ -44,27 +44,3 @@ Attic Patches and Suggestions
 - Johann Klähn
 - Petros Moisiadis
 - Thomas Waldmann
-
-BLAKE2
-------
-
-Borg includes BLAKE2: Copyright 2012, Samuel Neves <sneves@dei.uc.pt>, licensed under the terms
-of the CC0, the OpenSSL Licence, or the Apache Public License 2.0.
-
-Slicing CRC32
--------------
-
-Borg includes a fast slice-by-8 implementation of CRC32, Copyright 2011-2015 Stephan Brumme,
-licensed under the terms of a zlib license. See http://create.stephan-brumme.com/crc32/
-
-Folding CRC32
--------------
-
-Borg includes an extremely fast folding implementation of CRC32, Copyright 2013 Intel Corporation,
-licensed under the terms of the zlib license.
-
-xxHash
-------
-
-XXH64, a fast non-cryptographic hash algorithm. Copyright 2012-2016 Yann Collet,
-licensed under a BSD 2-clause license.

+ 16 - 6
docs/changes.rst

@@ -61,12 +61,12 @@ Compatibility notes:
 
       - borg create ARCHIVE ...
       - borg list ARCHIVE
+      - borg extract ARCHIVE ...
+      - borg diff ARCH1 ARCH2
+      - borg rename OLDNAME NEWNAME
       - borg info -a ARCH_GLOB
       - borg delete -a ARCH_GLOB
-      - borg diff ARCH1 ARCH2
-      - borg extract ARCHIVE ...
       - borg recreate -a ARCH_GLOB ...
-      - borg rename OLDNAME NEWNAME
       - borg mount -a ARCH_GLOB mountpoint ...
 
     For more details, please consult the docs or --help option output.
@@ -95,11 +95,15 @@ Changes:
   - symmetric hlid (all hardlinks pointing to same inode have same hlid)
   - all archived hardlinked regular files have a chunks list
 
-- borg init --other-repo=OTHER_REPO: reuse key material from OTHER_REPO, #6554
+- borg init --other-repo=OTHER_REPO: reuse key material from OTHER_REPO, #6554.
+  This is useful if you want to use borg transfer to transfer archives from an
+  existing borg 1.1/1.2 repo. If the chunker secret and the id key and algorithm
+  stay the same, the deduplication will also work between past and future backups.
+
 - borg transfer:
 
-  - efficiently copy archives from borg 1.1/1.2 repo to new repo. uses
-    deduplication and does not decompress/recompress file content data.
+  - efficiently copy archives from a borg 1.1/1.2 repo to a new repo.
+    uses deduplication and does not decompress/recompress file content data.
   - does some cleanups / fixes / conversions:
 
     - disallow None value for .user/group/chunks/chunks_healthy
@@ -111,9 +115,15 @@ Changes:
     - all hardlinks have chunks, maybe chunks_healty, hlid
     - remove the zlib type bytes hack
     - make sure items with chunks have precomputed size
+    - removes the csize element from the tuples in the Item.chunks list
     - clean item of attic 0.13 'acl' bug remnants
 
 - crypto: see 1.3.0a1 log entry
+- removed "borg upgrade" command (not needed any more)
+- compact: removed --cleanup-commits option
+- docs: fixed quickstart and usage docs with new cli command syntax
+- docs: removed the parts talking about AES-CTR mode issues (we will not
+  use that any more)
 
 
 Version 1.3.0a1 (2022-04-15)

+ 0 - 1
docs/usage.rst

@@ -60,7 +60,6 @@ Usage
    usage/tar
 
    usage/transfer
-   usage/upgrade
    usage/benchmark
 
    usage/help

+ 29 - 0
docs/usage/transfer.rst

@@ -1 +1,30 @@
 .. include:: transfer.rst.inc
+
+Examples
+~~~~~~~~
+::
+    # 0. Have borg 2.0 installed on client AND server, have a b12 repo copy for testing.
+
+    # 1. Create a new "related" repository:
+    # here, the existing borg 1.2 repo used repokey-blake2 (and aes-ctr mode),
+    # thus we use repokey-blake2-aes-ocb for the new borg 2.0 repo.
+    # staying with the same chunk id algorithm (blake2) and with the same
+    # key material (via --other-repo <oldrepo>) will make deduplication work
+    # between old archives (copied with borg transfer) and future ones.
+    # the AEAD cipher does not matter (everything must be re-encrypted and
+    # re-authenticated anyway), you could also choose repokey-blake2-chacha20-poly1305.
+    # in case your old borg repo did not use blake2, just remove the "-blake2".
+    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 rcreate \
+           --other-repo ssh://borg2@borgbackup/./tests/b12 -e repokey-blake2-aes-ocb
+
+    # 2. Check what and how much it would transfer:
+    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer \
+           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
+
+    # 3. Transfer (copy) archives from old repo into new repo (takes time and space!):
+    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer \
+           --other-repo ssh://borg2@borgbackup/./tests/b12
+
+    # 4. Check if we have everything (same as 2.):
+    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer \
+           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run

+ 0 - 16
docs/usage/upgrade.rst

@@ -1,16 +0,0 @@
-.. include:: upgrade.rst.inc
-
-Examples
-~~~~~~~~
-::
-
-    # Upgrade the borg repository to the most recent version.
-    $ borg upgrade -v /path/to/repo
-    making a hardlink copy in /path/to/repo.before-upgrade-2016-02-15-20:51:55
-    opening attic repository with borg and converting
-    no key file found for repository
-    converting repo index /path/to/repo/index.0
-    converting 1 segments...
-    converting borg 0.xx to borg current
-    no key file found for repository
-

+ 0 - 106
docs/usage/upgrade.rst.inc

@@ -1,106 +0,0 @@
-.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
-
-.. _borg_upgrade:
-
-borg upgrade
-------------
-.. code-block:: none
-
-    borg [common options] upgrade [options]
-
-.. only:: html
-
-    .. class:: borg-options-table
-
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    | **optional arguments**                                                                                                                                                         |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    |                                                       | ``-n``, ``--dry-run`` | do not change repository                                                                       |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    |                                                       | ``--inplace``         | rewrite repository in place, with no chance of going back to older versions of the repository. |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    |                                                       | ``--force``           | Force upgrade                                                                                  |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    |                                                       | ``--tam``             | Enable manifest authentication (in key and cache) (Borg 1.0.9 and later).                      |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    |                                                       | ``--disable-tam``     | Disable manifest authentication (in key and cache).                                            |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-    | .. class:: borg-common-opt-ref                                                                                                                                                 |
-    |                                                                                                                                                                                |
-    | :ref:`common_options`                                                                                                                                                          |
-    +-------------------------------------------------------+-----------------------+------------------------------------------------------------------------------------------------+
-
-    .. raw:: html
-
-        <script type='text/javascript'>
-        $(document).ready(function () {
-            $('.borg-options-table colgroup').remove();
-        })
-        </script>
-
-.. only:: latex
-
-
-
-    optional arguments
-        -n, --dry-run     do not change repository
-        --inplace         rewrite repository in place, with no chance of going back to older versions of the repository.
-        --force           Force upgrade
-        --tam             Enable manifest authentication (in key and cache) (Borg 1.0.9 and later).
-        --disable-tam     Disable manifest authentication (in key and cache).
-
-
-    :ref:`common_options`
-        |
-
-Description
-~~~~~~~~~~~
-
-Upgrade an existing, local Borg repository.
-
-When you do not need borg upgrade
-+++++++++++++++++++++++++++++++++
-
-Not every change requires that you run ``borg upgrade``.
-
-You do **not** need to run it when:
-
-- moving your repository to a different place
-- upgrading to another point release (like 1.0.x to 1.0.y),
-  except when noted otherwise in the changelog
-- upgrading from 1.0.x to 1.1.x,
-  except when noted otherwise in the changelog
-
-Borg 1.x.y upgrades
-+++++++++++++++++++
-
-Use ``borg upgrade --tam REPO`` to require manifest authentication
-introduced with Borg 1.0.9 to address security issues. This means
-that modifying the repository after doing this with a version prior
-to 1.0.9 will raise a validation error, so only perform this upgrade
-after updating all clients using the repository to 1.0.9 or newer.
-
-This upgrade should be done on each client for safety reasons.
-
-If a repository is accidentally modified with a pre-1.0.9 client after
-this upgrade, use ``borg upgrade --tam --force REPO`` to remedy it.
-
-If you routinely do this you might not want to enable this upgrade
-(which will leave you exposed to the security issue). You can
-reverse the upgrade by issuing ``borg upgrade --disable-tam REPO``.
-
-See
-https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability
-for details.
-
-Borg 0.xx to Borg 1.x
-+++++++++++++++++++++
-
-This currently supports converting Borg 0.xx to 1.0.
-
-Currently, only LOCAL repositories can be upgraded (issue #465).
-
-Please note that ``borg create`` (since 1.0.0) uses bigger chunks by
-default than old borg did, so the new chunks won't deduplicate
-with the old chunks in the upgraded repository.
-See ``--chunker-params`` option of ``borg create`` and ``borg recreate``.

+ 1 - 136
src/borg/archiver.py

@@ -490,16 +490,6 @@ class Archiver:
         if key.tam_required:
             tam_file = tam_required_file(repository)
             open(tam_file, 'w').close()
-            logger.warning(
-                '\n'
-                'By default repositories initialized with this version will produce security\n'
-                'errors if written to with an older version (up to and including Borg 1.0.8).\n'
-                '\n'
-                'If you want to use these older versions, you can disable the check by running:\n'
-                'borg upgrade --disable-tam %s\n'
-                '\n'
-                'See https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability '
-                'for details about the security implications.', shlex.quote(path))
 
         if key.NAME != 'plaintext':
             logger.warning(
@@ -1892,56 +1882,6 @@ class Archiver:
                           logger=logging.getLogger('borg.output.stats'))
         return self.exit_code
 
-    @with_repository(fake=('tam', 'disable_tam'), invert_fake=True, manifest=False, exclusive=True)
-    def do_upgrade(self, args, repository, manifest=None, key=None):
-        """upgrade a repository from a previous version"""
-        if args.tam:
-            manifest, key = Manifest.load(repository, (Manifest.Operation.CHECK,), force_tam_not_required=args.force)
-
-            if not hasattr(key, 'change_passphrase'):
-                print('This repository is not encrypted, cannot enable TAM.')
-                return EXIT_ERROR
-
-            if not manifest.tam_verified or not manifest.config.get('tam_required', False):
-                # The standard archive listing doesn't include the archive ID like in borg 1.1.x
-                print('Manifest contents:')
-                for archive_info in manifest.archives.list(sort_by=['ts']):
-                    print(format_archive(archive_info), '[%s]' % bin_to_hex(archive_info.id))
-                manifest.config['tam_required'] = True
-                manifest.write()
-                repository.commit(compact=False)
-            if not key.tam_required:
-                key.tam_required = True
-                key.change_passphrase(key._passphrase)
-                print('Key updated')
-                if hasattr(key, 'find_key'):
-                    print('Key location:', key.find_key())
-            if not tam_required(repository):
-                tam_file = tam_required_file(repository)
-                open(tam_file, 'w').close()
-                print('Updated security database')
-        elif args.disable_tam:
-            manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK, force_tam_not_required=True)
-            if tam_required(repository):
-                os.unlink(tam_required_file(repository))
-            if key.tam_required:
-                key.tam_required = False
-                key.change_passphrase(key._passphrase)
-                print('Key updated')
-                if hasattr(key, 'find_key'):
-                    print('Key location:', key.find_key())
-            manifest.config['tam_required'] = False
-            manifest.write()
-            repository.commit(compact=False)
-        else:
-            # mainly for upgrades from borg 0.xx -> 1.0.
-            repo = BorgRepositoryUpgrader(args.location.path, create=False)
-            try:
-                repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress)
-            except NotImplementedError as e:
-                print("warning: %s" % e)
-        return self.exit_code
-
     @with_repository(cache=True, exclusive=True, compatibility=(Manifest.Operation.CHECK,))
     def do_recreate(self, args, repository, manifest, key, cache):
         """Re-create archives"""
@@ -2093,7 +2033,7 @@ class Archiver:
         data = repository.get(Manifest.MANIFEST_ID)
         repository.put(Manifest.MANIFEST_ID, data)
         threshold = args.threshold / 100
-        repository.commit(compact=True, threshold=threshold, cleanup_commits=args.cleanup_commits)
+        repository.commit(compact=True, threshold=threshold)
         return EXIT_SUCCESS
 
     @with_repository(exclusive=True, manifest=False)
@@ -3536,11 +3476,6 @@ class Archiver:
         given by the ``--threshold`` option. If omitted, a threshold of 10% is used.
         When using ``--verbose``, borg will output an estimate of the freed space.
 
-        After upgrading borg (server) to 1.2+, you can use ``borg compact --cleanup-commits``
-        to clean up the numerous 17byte commit-only segments that borg 1.1 did not clean up
-        due to a bug. It is enough to do that once per repository. After cleaning up the
-        commits, borg will also do a normal compaction.
-
         See :ref:`separate_compaction` in Additional Notes for more details.
         """)
         subparser = subparsers.add_parser('compact', parents=[common_parser], add_help=False,
@@ -3549,8 +3484,6 @@ class Archiver:
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='compact segment files / free space in repo')
         subparser.set_defaults(func=self.do_compact)
-        subparser.add_argument('--cleanup-commits', dest='cleanup_commits', action='store_true',
-                               help='cleanup commit-only 17-byte segment files')
         subparser.add_argument('--threshold', metavar='PERCENT', dest='threshold',
                                type=int, default=10,
                                help='set minimum threshold for saved space in PERCENT (Default: 10)')
@@ -5077,74 +5010,6 @@ class Archiver:
         subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str,
                                help='mountpoint of the filesystem to umount')
 
-        # borg upgrade
-        upgrade_epilog = process_epilog("""
-        Upgrade an existing, local Borg repository.
-
-        When you do not need borg upgrade
-        +++++++++++++++++++++++++++++++++
-
-        Not every change requires that you run ``borg upgrade``.
-
-        You do **not** need to run it when:
-
-        - moving your repository to a different place
-        - upgrading to another point release (like 1.0.x to 1.0.y),
-          except when noted otherwise in the changelog
-        - upgrading from 1.0.x to 1.1.x,
-          except when noted otherwise in the changelog
-
-        Borg 1.x.y upgrades
-        +++++++++++++++++++
-
-        Use ``borg upgrade --tam REPO`` to require manifest authentication
-        introduced with Borg 1.0.9 to address security issues. This means
-        that modifying the repository after doing this with a version prior
-        to 1.0.9 will raise a validation error, so only perform this upgrade
-        after updating all clients using the repository to 1.0.9 or newer.
-
-        This upgrade should be done on each client for safety reasons.
-
-        If a repository is accidentally modified with a pre-1.0.9 client after
-        this upgrade, use ``borg upgrade --tam --force REPO`` to remedy it.
-
-        If you routinely do this you might not want to enable this upgrade
-        (which will leave you exposed to the security issue). You can
-        reverse the upgrade by issuing ``borg upgrade --disable-tam REPO``.
-
-        See
-        https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability
-        for details.
-
-        Borg 0.xx to Borg 1.x
-        +++++++++++++++++++++
-
-        This currently supports converting Borg 0.xx to 1.0.
-
-        Currently, only LOCAL repositories can be upgraded (issue #465).
-
-        Please note that ``borg create`` (since 1.0.0) uses bigger chunks by
-        default than old borg did, so the new chunks won't deduplicate
-        with the old chunks in the upgraded repository.
-        See ``--chunker-params`` option of ``borg create`` and ``borg recreate``.""")
-        subparser = subparsers.add_parser('upgrade', parents=[common_parser], add_help=False,
-                                          description=self.do_upgrade.__doc__,
-                                          epilog=upgrade_epilog,
-                                          formatter_class=argparse.RawDescriptionHelpFormatter,
-                                          help='upgrade repository format')
-        subparser.set_defaults(func=self.do_upgrade)
-        subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true',
-                               help='do not change repository')
-        subparser.add_argument('--inplace', dest='inplace', action='store_true',
-                               help='rewrite repository in place, with no chance of going back '
-                                    'to older versions of the repository.')
-        subparser.add_argument('--force', dest='force', action='store_true',
-                               help='Force upgrade')
-        subparser.add_argument('--tam', dest='tam', action='store_true',
-                               help='Enable manifest authentication (in key and cache) (Borg 1.0.9 and later).')
-        subparser.add_argument('--disable-tam', dest='disable_tam', action='store_true',
-                               help='Disable manifest authentication (in key and cache).')
-
         # borg with-lock
         with_lock_epilog = process_epilog("""
         This command runs a user-specified command while the repository lock is held.

+ 1 - 6
src/borg/crypto/key.py

@@ -61,12 +61,7 @@ class UnsupportedKeyFormatError(Error):
 
 class TAMRequiredError(IntegrityError):
     __doc__ = textwrap.dedent("""
-    Manifest is unauthenticated, but it is required for this repository.
-
-    This either means that you are under attack, or that you modified this repository
-    with a Borg version older than 1.0.9 after TAM authentication was enabled.
-
-    In the latter case, use "borg upgrade --tam --force '{}'" to re-authenticate the manifest.
+    Manifest is unauthenticated, but it is required for this repository. Is somebody attacking you?
     """).strip()
     traceback = False
 

+ 2 - 3
src/borg/remote.py

@@ -905,9 +905,8 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
 
     @api(since=parse_version('1.0.0'),
          compact={'since': parse_version('1.2.0a0'), 'previously': True, 'dontcare': True},
-         threshold={'since': parse_version('1.2.0a8'), 'previously': 0.1, 'dontcare': True},
-         cleanup_commits={'since': parse_version('1.2.0a0'), 'previously': False, 'dontcare': True})
-    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
+         threshold={'since': parse_version('1.2.0a8'), 'previously': 0.1, 'dontcare': True})
+    def commit(self, save_space=False, compact=True, threshold=0.1):
         """actual remoting is done via self.call in the @api decorator"""
 
     @api(since=parse_version('1.0.0'))

+ 2 - 9
src/borg/repository.py

@@ -153,7 +153,7 @@ class Repository:
         """{} does not have a valid configuration. Check repo config [{}]."""
 
     class AtticRepository(Error):
-        """Attic repository detected. Please use borg < 1.3 to run "borg upgrade {}"."""
+        """Attic repository detected. Please use borg <= 1.2 to run "borg upgrade {}"."""
 
     class CheckNeeded(ErrorWithTraceback):
         """Inconsistency detected. Please run "borg check {}"."""
@@ -495,7 +495,7 @@ class Repository:
             self.lock.release()
             self.lock = None
 
-    def commit(self, save_space=False, compact=True, threshold=0.1, cleanup_commits=False):
+    def commit(self, save_space=False, compact=True, threshold=0.1):
         """Commit transaction
         """
         # save_space is not used anymore, but stays for RPC/API compatibility.
@@ -509,13 +509,6 @@ class Repository:
         self.segments.setdefault(segment, 0)
         self.compact[segment] += LoggedIO.header_fmt.size
         if compact and not self.append_only:
-            if cleanup_commits:
-                # due to bug #2850, there might be a lot of commit-only segment files.
-                # this is for a one-time cleanup of these 17byte files.
-                for segment, filename in self.io.segment_iterator():
-                    if os.path.getsize(filename) == 17:
-                        self.segments[segment] = 0
-                        self.compact[segment] = LoggedIO.header_fmt.size
             self.compact_segments(threshold)
         self.write_index()
         self.rollback()

+ 0 - 34
src/borg/testsuite/archiver.py

@@ -3888,21 +3888,6 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
         self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=repokey')
         self.create_src_archive('archive1234')
         repository = Repository(self.repository_path, exclusive=True)
-        with repository:
-            shutil.rmtree(get_security_dir(bin_to_hex(repository.id)))
-            _, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
-            key.tam_required = False
-            key.change_passphrase(key._passphrase)
-
-            manifest = msgpack.unpackb(key.decrypt(Manifest.MANIFEST_ID, repository.get(Manifest.MANIFEST_ID)))
-            del manifest['tam']
-            repository.put(Manifest.MANIFEST_ID, key.encrypt(Manifest.MANIFEST_ID, msgpack.packb(manifest)))
-            repository.commit(compact=False)
-        output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--debug')
-        assert 'archive1234' in output
-        assert 'TAM not found and not required' in output
-        # Run upgrade
-        self.cmd(f'--repo={self.repository_location}', 'upgrade', '--tam')
         # Manifest must be authenticated now
         output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--debug')
         assert 'archive1234' in output
@@ -3912,25 +3897,6 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
         # Fails
         with pytest.raises(TAMRequiredError):
             self.cmd(f'--repo={self.repository_location}', 'rlist')
-        # Force upgrade
-        self.cmd(f'--repo={self.repository_location}', 'upgrade', '--tam', '--force')
-        self.cmd(f'--repo={self.repository_location}', 'rlist')
-
-    def test_disable(self):
-        self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=repokey')
-        self.create_src_archive('archive1234')
-        self.cmd(f'--repo={self.repository_location}', 'upgrade', '--disable-tam')
-        repository = Repository(self.repository_path, exclusive=True)
-        self.spoof_manifest(repository)
-        assert not self.cmd(f'--repo={self.repository_location}', 'rlist')
-
-    def test_disable2(self):
-        self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=repokey')
-        self.create_src_archive('archive1234')
-        repository = Repository(self.repository_path, exclusive=True)
-        self.spoof_manifest(repository)
-        self.cmd(f'--repo={self.repository_location}', 'upgrade', '--disable-tam')
-        assert not self.cmd(f'--repo={self.repository_location}', 'rlist')
 
 
 class RemoteArchiverTestCase(ArchiverTestCase):