pybiolib 1.1.988__tar.gz → 1.1.997__tar.gz

{pybiolib-1.1.988 → pybiolib-1.1.997}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.988
+Version: 1.1.997
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.988 → pybiolib-1.1.997}/biolib/biolib_binary_format/remote_endpoints.py

@@ -1,6 +1,8 @@
 import datetime
 from urllib.parse import urlparse, parse_qs
 
+from biolib.biolib_logging import logger
+
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format.utils import RemoteEndpoint
 
@@ -23,5 +25,6 @@ class RemoteJobStorageResultEndpoint(RemoteEndpoint):
             query_params = parse_qs(parsed_url.query)
             time_at_generation = datetime.datetime.strptime(query_params['X-Amz-Date'][0], '%Y%m%dT%H%M%SZ')
             self._expires_at = time_at_generation + datetime.timedelta(seconds=int(query_params['X-Amz-Expires'][0]))
+            logger.debug(f'Job "{self._job_id}" fetched presigned URL with expiry at {self._expires_at.isoformat()}')
 
         return self.presigned_url

{pybiolib-1.1.988 → pybiolib-1.1.997}/biolib/biolib_binary_format/utils.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-
 import io
+import time
 
 import requests
+from requests import Response
 
-from biolib.typing_utils import Iterable
+from biolib.biolib_logging import logger
+from biolib.typing_utils import Iterable, Optional, Dict
 from biolib.utils.multipart_uploader import get_chunk_iterator_from_bytes
 
 
@@ -49,6 +51,46 @@ class IndexableBuffer(ABC):
         return data
 
 
+def _get_request_with_retries_for_timeout(
+        url: str,
+        headers: Optional[Dict[str, str]] = None,
+        stream: bool = False,
+) -> Response:
+    retries = 10
+    for retry_count in range(retries):
+        if retry_count > 0:
+            time.sleep(1 * retry_count)
+            logger.debug('Retrying HTTP GET request...')
+
+        try:
+            response = requests.get(
+                headers=headers,
+                stream=stream,
+                timeout=10,
+                url=url,
+            )
+        except requests.exceptions.ReadTimeout:
+            logger.debug('Got read timeout of HTTP GET request')
+            continue
+
+        if response.status_code == 502:
+            logger.debug('Got 502 status for HTTP GET')
+            continue
+
+        if response.status_code == 400:
+            logger.error(
+                f'GET request for "{url}" got response with status {response.status_code} and '
+                f'content: {response.content.decode()}'
+            )
+
+        if not response.ok:
+            raise Exception(f'GET request for "{url}" got response with status {response.status_code}')
+
+        return response
+
+    raise Exception(f'Failed to complete HTTP GET request for "{url}" after {retries} retries')
+
+
 class RemoteIndexableBuffer(IndexableBuffer):
 
     def __init__(self, endpoint: RemoteEndpoint):
@@ -63,9 +105,10 @@ class RemoteIndexableBuffer(IndexableBuffer):
             return bytes(0)
 
         end = start + length - 1
-        response = requests.get(url=self._endpoint.get_remote_url(), headers={'range': f'bytes={start}-{end}'})
-        if not response.ok:
-            raise Exception(f'get_data got not ok response status {response.status_code}')
+        response = _get_request_with_retries_for_timeout(
+            url=self._endpoint.get_remote_url(),
+            headers={'range': f'bytes={start}-{end}'},
+        )
 
         data: bytes = response.content
         if len(data) != length:
@@ -81,11 +124,10 @@ class RemoteIndexableBuffer(IndexableBuffer):
             return []
 
         end = start + length - 1
-        response = requests.get(
+        response = _get_request_with_retries_for_timeout(
             url=self._endpoint.get_remote_url(),
             headers={'range': f'bytes={start}-{end}'},
             stream=True,
-            timeout=60,
         )
         return response.iter_content(chunk_size=chunk_size)
 

{pybiolib-1.1.988 → pybiolib-1.1.997}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.1.988"
+version = "1.1.997"
 description = "BioLib Python Client"
 readme = "README.md"
 license = "MIT"