pybiolib 1.2.1379__tar.gz → 1.2.1389__tar.gz

{pybiolib-1.2.1379 → pybiolib-1.2.1389}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybiolib
-Version: 1.2.1379
+Version: 1.2.1389
 Summary: BioLib Python Client
 License: MIT
 License-File: LICENSE

pybiolib-1.2.1389/biolib/biolib_binary_format/remote_stream_seeker.py

@@ -0,0 +1,84 @@
+import time
+
+from biolib.biolib_binary_format.utils import IndexableBuffer
+from biolib.biolib_logging import logger
+from biolib.typing_utils import Iterable
+
+
+class StreamSeeker:
+    def __init__(
+            self,
+            upstream_buffer: IndexableBuffer,
+            files_data_start: int,
+            files_data_end: int,
+            max_chunk_size: int,
+    ):
+        self._upstream_buffer = upstream_buffer
+        self._files_data_end = files_data_end
+        self._download_chunk_size_in_bytes = 100_000
+        self._min_chunk_size = min(100_000, max_chunk_size)
+        self._max_chunk_size = max_chunk_size
+        self._target_download_time_seconds = 1.0
+
+        self._buffer_start = files_data_start
+        self._buffer = bytearray()
+
+    def seek_and_read(self, file_start: int, file_length: int) -> Iterable[bytes]:
+        assert file_start >= self._buffer_start
+        self._buffer = self._buffer[file_start - self._buffer_start:]  # Returns empty array if "out of bounds"
+        self._buffer_start = file_start
+
+        while True:
+            file_byte_count_remaining = file_length - (self._buffer_start - file_start)
+            if file_byte_count_remaining == 0:
+                return
+
+            start_of_fetch = self._buffer_start + len(self._buffer)
+            byte_count_left_in_stream = self._files_data_end - start_of_fetch
+
+            if byte_count_left_in_stream != 0:
+                # Only fetch if there is still data left upstream
+                if self._download_chunk_size_in_bytes > len(self._buffer):
+                    # Only fetch if size of buffer is below chunk size
+                    fetch_size = min(byte_count_left_in_stream, self._download_chunk_size_in_bytes)
+
+                    start_time = time.monotonic()
+                    fetched_data = self._upstream_buffer.get_data(
+                        start=start_of_fetch,
+                        length=fetch_size,
+                    )
+                    download_time = time.monotonic() - start_time
+
+                    self._buffer.extend(fetched_data)
+
+                    if download_time > 0:
+                        self._adjust_chunk_size(download_time, fetch_size)
+
+            bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
+            yield bytes_to_yield
+            self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
+            self._buffer_start += len(bytes_to_yield)
+
+    def _adjust_chunk_size(self, download_time: float, _bytes_downloaded: int) -> None:
+        new_chunk_size = self._download_chunk_size_in_bytes
+        time_ratio = download_time / self._target_download_time_seconds
+
+        if time_ratio > 1.1:
+            adjustment_factor = 1.0 / time_ratio
+            adjustment_factor = max(adjustment_factor, 0.5)
+            new_chunk_size = int(self._download_chunk_size_in_bytes * adjustment_factor)
+        elif time_ratio < 0.9:
+            adjustment_factor = 1.0 / time_ratio
+            new_chunk_size = int(self._download_chunk_size_in_bytes * adjustment_factor)
+
+        new_chunk_size = max(
+            self._min_chunk_size,
+            min(self._max_chunk_size, new_chunk_size)
+        )
+
+        if new_chunk_size != self._download_chunk_size_in_bytes:
+            logger.debug(
+                f"Adjusting chunk size: {self._download_chunk_size_in_bytes} -> {new_chunk_size} bytes "
+                f"(download_time={download_time:.2f}s, time_ratio={time_ratio:.2f})"
+            )
+            self._download_chunk_size_in_bytes = new_chunk_size

{pybiolib-1.2.1379 → pybiolib-1.2.1389}/biolib/jobs/job_result.py

@@ -55,7 +55,7 @@ class JobResult:
         stream_seeker = StreamSeeker(
             files_data_start=first_file.start,
             files_data_end=last_file.start + last_file.length,
-            download_chunk_size_in_bytes=min(total_files_data_to_download_in_bytes, 10_000_000),
+            max_chunk_size=min(total_files_data_to_download_in_bytes, 10_000_000),
             upstream_buffer=module_output.buffer,
         )
 

{pybiolib-1.2.1379 → pybiolib-1.2.1389}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.2.1379"
+version = "1.2.1389"
 description = "BioLib Python Client"
 readme = "PYPI_README.md"
 license = "MIT"

pybiolib-1.2.1379/biolib/biolib_binary_format/remote_stream_seeker.py

@@ -1,45 +0,0 @@
-from biolib.biolib_binary_format.utils import IndexableBuffer
-from biolib.typing_utils import Iterable
-
-
-class StreamSeeker:
-    def __init__(
-            self,
-            upstream_buffer: IndexableBuffer,
-            files_data_start: int,
-            files_data_end: int,
-            download_chunk_size_in_bytes: int,
-    ):
-        self._upstream_buffer = upstream_buffer
-        self._files_data_end = files_data_end
-        self._download_chunk_size_in_bytes = download_chunk_size_in_bytes
-
-        self._buffer_start = files_data_start
-        self._buffer = bytearray()
-
-    def seek_and_read(self, file_start: int, file_length: int) -> Iterable[bytes]:
-        assert file_start >= self._buffer_start
-        self._buffer = self._buffer[file_start - self._buffer_start:]  # Returns empty array if "out of bounds"
-        self._buffer_start = file_start
-
-        while True:
-            file_byte_count_remaining = file_length - (self._buffer_start - file_start)
-            if file_byte_count_remaining == 0:
-                return
-
-            start_of_fetch = self._buffer_start + len(self._buffer)
-            byte_count_left_in_stream = self._files_data_end - start_of_fetch
-
-            if byte_count_left_in_stream != 0:
-                # Only fetch if there is still data left upstream
-                if self._download_chunk_size_in_bytes > len(self._buffer):
-                    # Only fetch if size of buffer is below chunk size
-                    self._buffer.extend(self._upstream_buffer.get_data(
-                        start=start_of_fetch,
-                        length=min(byte_count_left_in_stream, self._download_chunk_size_in_bytes),
-                    ))
-
-            bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
-            yield bytes_to_yield
-            self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
-            self._buffer_start += len(bytes_to_yield)