Sfoglia il codice sorgente

fetch_many: add replacement_chunk=True param

Sometimes, usually for file content chunks, it makes sense to
generate all-zero replacement chunks on-the-fly.

But for e.g. an archive items metadata stream, this does not
make sense (because it wants to msgpack.unpack the data), so
we rather want None. In that case, we do not have the size
information anyway.
Thomas Waldmann 2 mesi fa
parent
commit
3fdf7a3e78
1 ha cambiato i file con 10 aggiunte e 4 eliminazioni
  1. 10 4
      src/borg/archive.py

+ 10 - 4
src/borg/archive.py

@@ -273,7 +273,9 @@ class DownloadPipeline:
         """
         """
         self.hlids_preloaded = set()
         self.hlids_preloaded = set()
         unpacker = msgpack.Unpacker(use_list=False)
         unpacker = msgpack.Unpacker(use_list=False)
-        for data in self.fetch_many(ids, ro_type=ROBJ_ARCHIVE_STREAM):
+        for data in self.fetch_many(ids, ro_type=ROBJ_ARCHIVE_STREAM, replacement_chunk=False):
+            if data is None:
+                continue  # archive stream chunk missing
             unpacker.feed(data)
             unpacker.feed(data)
             for _item in unpacker:
             for _item in unpacker:
                 item = Item(internal_dict=_item)
                 item = Item(internal_dict=_item)
@@ -312,7 +314,7 @@ class DownloadPipeline:
                 self.repository.preload([c.id for c in item.chunks])
                 self.repository.preload([c.id for c in item.chunks])
         return preload_chunks
         return preload_chunks
 
 
-    def fetch_many(self, chunks, is_preloaded=False, ro_type=None):
+    def fetch_many(self, chunks, is_preloaded=False, ro_type=None, replacement_chunk=True):
         assert ro_type is not None
         assert ro_type is not None
         ids = []
         ids = []
         sizes = []
         sizes = []
@@ -329,8 +331,12 @@ class DownloadPipeline:
             ids, sizes, self.repository.get_many(ids, is_preloaded=is_preloaded, raise_missing=False)
             ids, sizes, self.repository.get_many(ids, is_preloaded=is_preloaded, raise_missing=False)
         ):
         ):
             if cdata is None:
             if cdata is None:
-                logger.error(f"repository object {bin_to_hex(id)} missing, returning {size} zero bytes.")
-                data = zeros[:size] if size is not None else None
+                if replacement_chunk and size is not None:
+                    logger.error(f"repository object {bin_to_hex(id)} missing, returning {size} zero bytes.")
+                    data = zeros[:size]  # return an all-zero replacement chunk of correct size
+                else:
+                    logger.error(f"repository object {bin_to_hex(id)} missing, returning None.")
+                    data = None
             else:
             else:
                 _, data = self.repo_objs.parse(id, cdata, ro_type=ro_type)
                 _, data = self.repo_objs.parse(id, cdata, ro_type=ro_type)
             assert size is None or len(data) == size
             assert size is None or len(data) == size