|
@@ -194,6 +194,39 @@ class ChunkerFixed:
|
|
|
self.try_sparse = sparse and has_seek_hole
|
|
|
assert block_size <= len(zeros)
|
|
|
|
|
|
+ def _build_fmap(self, fd=None, fh=-1):
|
|
|
+ fmap = None
|
|
|
+ if self.try_sparse:
|
|
|
+ try:
|
|
|
+ if self.header_size > 0:
|
|
|
+ header_map = [(0, self.header_size, True), ]
|
|
|
+ dseek(self.header_size, os.SEEK_SET, fd, fh)
|
|
|
+ body_map = list(sparsemap(fd, fh))
|
|
|
+ dseek(0, os.SEEK_SET, fd, fh)
|
|
|
+ else:
|
|
|
+ header_map = []
|
|
|
+ body_map = list(sparsemap(fd, fh))
|
|
|
+ except OSError as err:
|
|
|
+ # seeking did not work
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ fmap = header_map + body_map
|
|
|
+
|
|
|
+ if fmap is None:
|
|
|
+ # either sparse processing (building the fmap) was not tried or it failed.
|
|
|
+ # in these cases, we just build a "fake fmap" that considers the whole file
|
|
|
+ # as range(s) of data (no holes), so we can use the same code.
|
|
|
+ # we build different fmaps here for the purpose of correct block alignment
|
|
|
+ # with or without a header block (of potentially different size).
|
|
|
+ if self.header_size > 0:
|
|
|
+ header_map = [(0, self.header_size, True), ]
|
|
|
+ body_map = [(self.header_size, 2 ** 62, True), ]
|
|
|
+ else:
|
|
|
+ header_map = []
|
|
|
+ body_map = [(0, 2 ** 62, True), ]
|
|
|
+ fmap = header_map + body_map
|
|
|
+ return fmap
|
|
|
+
|
|
|
def chunkify(self, fd=None, fh=-1, fmap=None):
|
|
|
"""
|
|
|
Cut a file into chunks.
|
|
@@ -203,37 +236,7 @@ class ChunkerFixed:
|
|
|
defaults to -1 which means not to use OS-level fd.
|
|
|
:param fmap: a file map, same format as generated by sparsemap
|
|
|
"""
|
|
|
- if fmap is None:
|
|
|
- if self.try_sparse:
|
|
|
- try:
|
|
|
- if self.header_size > 0:
|
|
|
- header_map = [(0, self.header_size, True), ]
|
|
|
- dseek(self.header_size, os.SEEK_SET, fd, fh)
|
|
|
- body_map = list(sparsemap(fd, fh))
|
|
|
- dseek(0, os.SEEK_SET, fd, fh)
|
|
|
- else:
|
|
|
- header_map = []
|
|
|
- body_map = list(sparsemap(fd, fh))
|
|
|
- except OSError as err:
|
|
|
- # seeking did not work
|
|
|
- pass
|
|
|
- else:
|
|
|
- fmap = header_map + body_map
|
|
|
-
|
|
|
- if fmap is None:
|
|
|
- # either sparse processing (building the fmap) was not tried or it failed.
|
|
|
- # in these cases, we just build a "fake fmap" that considers the whole file
|
|
|
- # as range(s) of data (no holes), so we can use the same code.
|
|
|
- # we build different fmaps here for the purpose of correct block alignment
|
|
|
- # with or without a header block (of potentially different size).
|
|
|
- if self.header_size > 0:
|
|
|
- header_map = [(0, self.header_size, True), ]
|
|
|
- body_map = [(self.header_size, 2 ** 62, True), ]
|
|
|
- else:
|
|
|
- header_map = []
|
|
|
- body_map = [(0, 2 ** 62, True), ]
|
|
|
- fmap = header_map + body_map
|
|
|
-
|
|
|
+ fmap =self._build_fmap(fd, fh) if fmap is None else fmap
|
|
|
offset = 0
|
|
|
for range_start, range_size, is_data in fmap:
|
|
|
if range_start != offset:
|