pybiolib 1.2.1379__py3-none-any.whl → 1.2.1389__py3-none-any.whl

biolib/biolib_binary_format/remote_stream_seeker.py

@@ -1,4 +1,7 @@
+import time
+
 from biolib.biolib_binary_format.utils import IndexableBuffer
+from biolib.biolib_logging import logger
 from biolib.typing_utils import Iterable
 
 
@@ -8,11 +11,14 @@ class StreamSeeker:
             upstream_buffer: IndexableBuffer,
             files_data_start: int,
             files_data_end: int,
-            download_chunk_size_in_bytes: int,
+            max_chunk_size: int,
     ):
         self._upstream_buffer = upstream_buffer
         self._files_data_end = files_data_end
-        self._download_chunk_size_in_bytes = download_chunk_size_in_bytes
+        self._download_chunk_size_in_bytes = 100_000
+        self._min_chunk_size = min(100_000, max_chunk_size)
+        self._max_chunk_size = max_chunk_size
+        self._target_download_time_seconds = 1.0
 
         self._buffer_start = files_data_start
         self._buffer = bytearray()
@@ -34,12 +40,45 @@ class StreamSeeker:
                 # Only fetch if there is still data left upstream
                 if self._download_chunk_size_in_bytes > len(self._buffer):
                     # Only fetch if size of buffer is below chunk size
-                    self._buffer.extend(self._upstream_buffer.get_data(
+                    fetch_size = min(byte_count_left_in_stream, self._download_chunk_size_in_bytes)
+
+                    start_time = time.monotonic()
+                    fetched_data = self._upstream_buffer.get_data(
                         start=start_of_fetch,
-                        length=min(byte_count_left_in_stream, self._download_chunk_size_in_bytes),
-                    ))
+                        length=fetch_size,
+                    )
+                    download_time = time.monotonic() - start_time
+
+                    self._buffer.extend(fetched_data)
+
+                    if download_time > 0:
+                        self._adjust_chunk_size(download_time, fetch_size)
 
             bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
             yield bytes_to_yield
             self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
             self._buffer_start += len(bytes_to_yield)
+
+    def _adjust_chunk_size(self, download_time: float, _bytes_downloaded: int) -> None:
+        new_chunk_size = self._download_chunk_size_in_bytes
+        time_ratio = download_time / self._target_download_time_seconds
+
+        if time_ratio > 1.1:
+            adjustment_factor = 1.0 / time_ratio
+            adjustment_factor = max(adjustment_factor, 0.5)
+            new_chunk_size = int(self._download_chunk_size_in_bytes * adjustment_factor)
+        elif time_ratio < 0.9:
+            adjustment_factor = 1.0 / time_ratio
+            new_chunk_size = int(self._download_chunk_size_in_bytes * adjustment_factor)
+
+        new_chunk_size = max(
+            self._min_chunk_size,
+            min(self._max_chunk_size, new_chunk_size)
+        )
+
+        if new_chunk_size != self._download_chunk_size_in_bytes:
+            logger.debug(
+                f"Adjusting chunk size: {self._download_chunk_size_in_bytes} -> {new_chunk_size} bytes "
+                f"(download_time={download_time:.2f}s, time_ratio={time_ratio:.2f})"
+            )
+            self._download_chunk_size_in_bytes = new_chunk_size

biolib/jobs/job_result.py

@@ -55,7 +55,7 @@ class JobResult:
         stream_seeker = StreamSeeker(
             files_data_start=first_file.start,
             files_data_end=last_file.start + last_file.length,
-            download_chunk_size_in_bytes=min(total_files_data_to_download_in_bytes, 10_000_000),
+            max_chunk_size=min(total_files_data_to_download_in_bytes, 10_000_000),
             upstream_buffer=module_output.buffer,
         )
 

{pybiolib-1.2.1379.dist-info → pybiolib-1.2.1389.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybiolib
-Version: 1.2.1379
+Version: 1.2.1389
 Summary: BioLib Python Client
 License: MIT
 License-File: LICENSE

{pybiolib-1.2.1379.dist-info → pybiolib-1.2.1389.dist-info}/RECORD

@@ -88,7 +88,7 @@ biolib/biolib_binary_format/file_in_container.py,sha256=j1eEPRxf_ew8I6G8sDiiZZxn
 biolib/biolib_binary_format/module_input.py,sha256=led2QhHeec_ymBPw5uEn3_3vJKI-1T8zrFQGqwEWLMY,2788
 biolib/biolib_binary_format/module_output_v2.py,sha256=J5ZO5gCSeudpE12EVDrjYrNTS2DwgszY-SVXT7Qjuyg,5913
 biolib/biolib_binary_format/remote_endpoints.py,sha256=V48mwOj3eAQAKp-8DjtWUdEKUyC0WKc1pEiKTmtjrJY,1651
-biolib/biolib_binary_format/remote_stream_seeker.py,sha256=uyi6kJBU1C1DWfiuR0kRUQIY7nalG7ocgwgngd3Ul4U,1999
+biolib/biolib_binary_format/remote_stream_seeker.py,sha256=QzCiDiVZMgX6HZTrw-oJsFIHzNd8V1AseaBZP9wqMxU,3608
 biolib/biolib_binary_format/saved_job.py,sha256=nFHVFRNTNcAFGODLSiBntCtMk55QKwreUq6qLX80dI4,1125
 biolib/biolib_binary_format/stdout_and_stderr.py,sha256=WfUUJFFCBrtfXjuWIaRPiWCpuBLxfko68oxoTKhrwx8,1023
 biolib/biolib_binary_format/system_exception.py,sha256=T3iL4_cSHAHim3RSDPS8Xyb1mfteaJBZonSXuRltc28,853
@@ -146,7 +146,7 @@ biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 biolib/experiments/experiment.py,sha256=ePipnJs6TW84jr9iBfYYRpYuEXUVGIxaD8Z8Jf0FYIA,13909
 biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
 biolib/jobs/job.py,sha256=X7F_jCwAwtPcI1qu-oLamZNzt1IVSIeOPWlax1ztfAM,29674
-biolib/jobs/job_result.py,sha256=_xqQu9z9BqPQrU6tjqKPuKlQDt5W0Zw5xiQvzEBkDyE,5266
+biolib/jobs/job_result.py,sha256=Yl22Z8dc-8iKeC7Y4q7kgd08Ndajys1W1MsAbvE8wOA,5252
 biolib/jobs/types.py,sha256=rFs6bQWsNI-nb1Hu9QzOW2zFZ8bOVt7ax4UpGVASxVA,1034
 biolib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/runtime/__init__.py,sha256=MlRepA11n2H-3plB5rzWyyHK2JmP6PiaP3i6x3vt0mg,506
@@ -161,8 +161,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=rImaghQGuIqTVWks6b9P2yKuN34uePUYPUFW_Wyoa4A,6737
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.1379.dist-info/METADATA,sha256=K_Bnjzo9XB9Aq5g63v7A6CEKrEc1YC5jh6u0gYMGqCA,1644
-pybiolib-1.2.1379.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-pybiolib-1.2.1379.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.1379.dist-info/licenses/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.1379.dist-info/RECORD,,
+pybiolib-1.2.1389.dist-info/METADATA,sha256=y8bhxtHgW135kkkPiKCE0WKPSZ4-DaEFS8aE9UAhmn0,1644
+pybiolib-1.2.1389.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+pybiolib-1.2.1389.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.1389.dist-info/licenses/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.1389.dist-info/RECORD,,