pybiolib 1.2.116.dev1__py3-none-any.whl → 1.2.122__py3-none-any.whl

biolib/biolib_binary_format/utils.py

@@ -1,6 +1,8 @@
 import io
 import math
+import multiprocessing
 from abc import ABC, abstractmethod
+from multiprocessing.pool import ThreadPool
 from typing import Callable, Optional
 
 from biolib._internal.http_client import HttpClient
@@ -155,12 +157,18 @@ class LazyLoadedFile:
         if self._length == 0:
             yield b''
         else:
-            chunk_size = 10_000_000
-            chunks_to_yield = math.ceil(self._length / chunk_size)
-            for chunk_idx in range(chunks_to_yield - 1):
-                yield self._buffer.get_data(start=self.start + chunk_idx * chunk_size, length=chunk_size)
+            chunk_size = 50_000_000
+            chunk_count = math.ceil(self._length / chunk_size)
+            chunk_indices_iterator = range(chunk_count - 1)
 
-            data_already_yielded = (chunks_to_yield - 1) * chunk_size
+            def get_chunk(chunk_index: int) -> bytes:
+                return self._buffer.get_data(start=self.start + chunk_index * chunk_size, length=chunk_size)
+
+            if chunk_count > 1:
+                with ThreadPool(processes=min(16, chunk_count, multiprocessing.cpu_count() - 1)) as pool:
+                    yield from pool.imap(func=get_chunk, iterable=chunk_indices_iterator)
+
+            data_already_yielded = (chunk_count - 1) * chunk_size
             yield self._buffer.get_data(
                 start=self.start + data_already_yielded,
                 length=self._length - data_already_yielded,

{pybiolib-1.2.116.dev1.dist-info → pybiolib-1.2.122.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.2.116.dev1
+Version: 1.2.122
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.2.116.dev1.dist-info → pybiolib-1.2.122.dist-info}/RECORD

@@ -53,7 +53,7 @@ biolib/biolib_binary_format/saved_job.py,sha256=nFHVFRNTNcAFGODLSiBntCtMk55QKwre
 biolib/biolib_binary_format/stdout_and_stderr.py,sha256=WfUUJFFCBrtfXjuWIaRPiWCpuBLxfko68oxoTKhrwx8,1023
 biolib/biolib_binary_format/system_exception.py,sha256=T3iL4_cSHAHim3RSDPS8Xyb1mfteaJBZonSXuRltc28,853
 biolib/biolib_binary_format/system_status_update.py,sha256=aOELuQ0k-GtpaZTUxYd0GFomP_OInmrK585y6fuQuKE,1191
-biolib/biolib_binary_format/utils.py,sha256=oVGnJlEdc2tmKaIjljBoZk18kDn-feGHu2HoeeHHD1c,5104
+biolib/biolib_binary_format/utils.py,sha256=jq31QEShs7TSrSKZBr8jif-bJAwVzQrqVxmymL6crpU,5453
 biolib/biolib_docker_client/__init__.py,sha256=aBfA6mtWSI5dBEfNNMD6bIZzCPloW4ghKm0wqQiljdo,1481
 biolib/biolib_download_container.py,sha256=8TmBV8iv3bCvkNlHa1SSsc4zl0wX_eaxhfnW5rvFIh8,1779
 biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
@@ -119,8 +119,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=WJnU9vZdwY8RHXvzATyV80OXzyJ7w9EkG33Tna9Nr6A,5698
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.116.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.116.dev1.dist-info/METADATA,sha256=IOA2DJfbIuqyiacUDW604dnqHy3TSyX6kqK4AHnVIC8,1512
-pybiolib-1.2.116.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.2.116.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.116.dev1.dist-info/RECORD,,
+pybiolib-1.2.122.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.122.dist-info/METADATA,sha256=ZqN7f5Rlsd29avkwSDiZmCoUi7OUiMlBqUmIl7OdDq8,1507
+pybiolib-1.2.122.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.2.122.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.122.dist-info/RECORD,,