feat(backend): enable s3 support using borgstore

Borg2 documentation mentions the support for the s3 backend however,
borg was missing the parsing bits for an s3 repo.

This updates the Location parser to parse the s3 url using the same
logic as borgstore.

Note: borgstore should be installed with the s3 dependencies in order
for the s3 backend to work.

Signed-off-by: Mike Mason <github@mikemrm.com>

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
 llfuse = ["llfuse >= 1.3.8"]
 pyfuse3 = ["pyfuse3 >= 3.1.1"]
 nofuse = []
+s3 = ["borgstore[s3] ~= 0.3.0"]
 
 [project.urls]
 "Homepage" = "https://borgbackup.org/"

src/borg/archiver/_common.py

@@ -37,7 +37,9 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_or_
             location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock, args=args
         )
 
-    elif location.proto in ("sftp", "file", "rclone") and not v1_or_v2:  # stuff directly supported by borgstore
+    elif (
+        location.proto in ("sftp", "file", "rclone", "s3", "b2") and not v1_or_v2
+    ):  # stuff directly supported by borgstore
         repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock)
 
     else:

src/borg/helpers/parseformat.py

@@ -472,6 +472,27 @@ class Location:
         re.VERBOSE,
     )
 
+    # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path
+    s3_re = re.compile(
+        r"""
+        (?P<s3type>(s3|b2)):
+        ((
+            (?P<profile>[^@:]+)  # profile (no colons allowed)
+            |
+            (?P<access_key_id>[^:@]+):(?P<access_key_secret>[^@]+)  # access key and secret
+        )@)?  # optional authentication
+        (
+            [^:/]+://  # scheme
+            (?P<hostname>[^:/]+)
+            (:(?P<port>\d+))?
+        )?  # optional endpoint
+        /
+        (?P<bucket>[^/]+)/  # bucket name
+        (?P<path>.+)  # path
+    """,
+        re.VERBOSE,
+    )
+
     rclone_re = re.compile(r"(?P<proto>rclone):(?P<path>(.*))", re.VERBOSE)
 
     file_or_socket_re = re.compile(r"(?P<proto>(file|socket))://" + abs_path_re, re.VERBOSE)
@@ -483,6 +504,7 @@ class Location:
         self.valid = False
         self.proto = None
         self.user = None
+        self._pass = None
         self._host = None
         self.port = None
         self.path = None
@@ -524,6 +546,15 @@ class Location:
             self.proto = m.group("proto")
             self.path = os.path.normpath(m.group("path"))
             return True
+        m = self.s3_re.match(text)
+        if m:
+            self.proto = m.group("s3type")
+            self.user = m.group("profile") if m.group("profile") else m.group("access_key_id")
+            self._pass = True if m.group("access_key_secret") else False
+            self._host = m.group("hostname")
+            self.port = m.group("port") and int(m.group("port")) or None
+            self.path = m.group("bucket") + "/" + m.group("path")
+            return True
         m = self.local_re.match(text)
         if m:
             self.proto = "file"
@@ -535,6 +566,7 @@ class Location:
         items = [
             "proto=%r" % self.proto,
             "user=%r" % self.user,
+            "pass=%r" % ("REDACTED" if self._pass else None),
             "host=%r" % self.host,
             "port=%r" % self.port,
             "path=%r" % self.path,
@@ -566,7 +598,7 @@ class Location:
             return self.path
         if self.proto == "rclone":
             return f"{self.proto}:{self.path}"
-        if self.proto in ("sftp", "ssh"):
+        if self.proto in ("sftp", "ssh", "s3", "b2"):
             return (
                 f"{self.proto}://"
                 f"{(self.user + '@') if self.user else ''}"

src/borg/testsuite/helpers/parseformat_test.py

@@ -86,30 +86,30 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("ssh://user@host:1234//absolute/path"))
-            == "Location(proto='ssh', user='user', host='host', port=1234, path='/absolute/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='/absolute/path')"
         )
         assert Location("ssh://user@host:1234//absolute/path").to_key_filename() == keys_dir + "host___absolute_path"
         assert (
             repr(Location("ssh://user@host:1234/relative/path"))
-            == "Location(proto='ssh', user='user', host='host', port=1234, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='relative/path')"
         )
         assert Location("ssh://user@host:1234/relative/path").to_key_filename() == keys_dir + "host__relative_path"
         assert (
             repr(Location("ssh://user@host/relative/path"))
-            == "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')"
         )
         assert (
             repr(Location("ssh://user@[::]:1234/relative/path"))
-            == "Location(proto='ssh', user='user', host='::', port=1234, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='::', port=1234, path='relative/path')"
         )
         assert Location("ssh://user@[::]:1234/relative/path").to_key_filename() == keys_dir + "____relative_path"
         assert (
             repr(Location("ssh://user@[::]/relative/path"))
-            == "Location(proto='ssh', user='user', host='::', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='::', port=None, path='relative/path')"
         )
         assert (
             repr(Location("ssh://user@[2001:db8::]:1234/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::', port=1234, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')"
         )
         assert (
             Location("ssh://user@[2001:db8::]:1234/relative/path").to_key_filename()
@@ -117,23 +117,23 @@ class TestLocationWithoutEnv:
         )
         assert (
             repr(Location("ssh://user@[2001:db8::]/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')"
         )
         assert (
             repr(Location("ssh://user@[2001:db8::c0:ffee]:1234/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=1234, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')"  # noqa: E501
         )
         assert (
             repr(Location("ssh://user@[2001:db8::c0:ffee]/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')"  # noqa: E501
         )
         assert (
             repr(Location("ssh://user@[2001:db8::192.0.2.1]:1234/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=1234, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')"  # noqa: E501
         )
         assert (
             repr(Location("ssh://user@[2001:db8::192.0.2.1]/relative/path"))
-            == "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')"  # noqa: E501
         )
         assert (
             Location("ssh://user@[2001:db8::192.0.2.1]/relative/path").to_key_filename()
@@ -141,20 +141,39 @@ class TestLocationWithoutEnv:
         )
         assert (
             repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path"))
-            == "Location(proto='ssh', user='user', "
+            == "Location(proto='ssh', user='user', pass=None, "
             "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')"
         )
         assert (
             repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path"))
-            == "Location(proto='ssh', user='user', "
+            == "Location(proto='ssh', user='user', pass=None, "
             "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')"
         )
 
+    def test_s3(self, monkeypatch, keys_dir):
+        monkeypatch.delenv("BORG_REPO", raising=False)
+        assert (
+            repr(Location("s3:/test/path"))
+            == "Location(proto='s3', user=None, pass=None, host=None, port=None, path='test/path')"
+        )
+        assert (
+            repr(Location("s3:profile@http://172.28.52.116:9000/test/path"))
+            == "Location(proto='s3', user='profile', pass=None, host='172.28.52.116', port=9000, path='test/path')"  # noqa: E501
+        )
+        assert (
+            repr(Location("s3:user:pass@http://172.28.52.116:9000/test/path"))
+            == "Location(proto='s3', user='user', pass='REDACTED', host='172.28.52.116', port=9000, path='test/path')"  # noqa: E501
+        )
+        assert (
+            repr(Location("b2:user:pass@https://s3.us-east-005.backblazeb2.com/test/path"))
+            == "Location(proto='b2', user='user', pass='REDACTED', host='s3.us-east-005.backblazeb2.com', port=None, path='test/path')"  # noqa: E501
+        )
+
     def test_rclone(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("rclone:remote:path"))
-            == "Location(proto='rclone', user=None, host=None, port=None, path='remote:path')"
+            == "Location(proto='rclone', user=None, pass=None, host=None, port=None, path='remote:path')"
         )
         assert Location("rclone:remote:path").to_key_filename() == keys_dir + "remote_path"
 
@@ -163,13 +182,13 @@ class TestLocationWithoutEnv:
         # relative path
         assert (
             repr(Location("sftp://user@host:1234/rel/path"))
-            == "Location(proto='sftp', user='user', host='host', port=1234, path='rel/path')"
+            == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='rel/path')"
         )
         assert Location("sftp://user@host:1234/rel/path").to_key_filename() == keys_dir + "host__rel_path"
         # absolute path
         assert (
             repr(Location("sftp://user@host:1234//abs/path"))
-            == "Location(proto='sftp', user='user', host='host', port=1234, path='/abs/path')"
+            == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='/abs/path')"
         )
         assert Location("sftp://user@host:1234//abs/path").to_key_filename() == keys_dir + "host___abs_path"
 
@@ -177,7 +196,7 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("socket:///repo/path"))
-            == "Location(proto='socket', user=None, host=None, port=None, path='/repo/path')"
+            == "Location(proto='socket', user=None, pass=None, host=None, port=None, path='/repo/path')"
         )
         assert Location("socket:///some/path").to_key_filename() == keys_dir + "_some_path"
 
@@ -185,11 +204,11 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("file:///some/path"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/some/path')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
         )
         assert (
             repr(Location("file:///some/path"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/some/path')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
         )
         assert Location("file:///some/path").to_key_filename() == keys_dir + "_some_path"
 
@@ -197,7 +216,7 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("file:////server/share/path"))
-            == "Location(proto='file', user=None, host=None, port=None, path='//server/share/path')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')"
         )
         assert Location("file:////server/share/path").to_key_filename() == keys_dir + "__server_share_path"
 
@@ -205,19 +224,22 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         rel_path = "path"
         abs_path = os.path.abspath(rel_path)
-        assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')"
+        assert (
+            repr(Location(rel_path))
+            == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')"
+        )
         assert Location("path").to_key_filename().endswith(rel_path)
 
     def test_abspath(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("/some/absolute/path"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/absolute/path')"
         )
         assert Location("/some/absolute/path").to_key_filename() == keys_dir + "_some_absolute_path"
         assert (
             repr(Location("/some/../absolute/path"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/absolute/path')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/absolute/path')"
         )
         assert Location("/some/../absolute/path").to_key_filename() == keys_dir + "_absolute_path"
 
@@ -226,11 +248,14 @@ class TestLocationWithoutEnv:
         # for a local path, borg creates a Location instance with an absolute path
         rel_path = "relative/path"
         abs_path = os.path.abspath(rel_path)
-        assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')"
+        assert (
+            repr(Location(rel_path))
+            == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')"
+        )
         assert Location(rel_path).to_key_filename().endswith("relative_path")
         assert (
             repr(Location("ssh://user@host/relative/path"))
-            == "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')"
+            == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')"
         )
         assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path"
 
@@ -238,17 +263,17 @@ class TestLocationWithoutEnv:
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("/abs/path:w:cols"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')"
         )
         assert Location("/abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols"
         assert (
             repr(Location("file:///abs/path:w:cols"))
-            == "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')"
+            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')"
         )
         assert Location("file:///abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols"
         assert (
             repr(Location("ssh://user@host/abs/path:w:cols"))
-            == "Location(proto='ssh', user='user', host='host', port=None, path='abs/path:w:cols')"
+            == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='abs/path:w:cols')"
         )
         assert Location("ssh://user@host/abs/path:w:cols").to_key_filename() == keys_dir + "host__abs_path_w_cols"