pybiolib 1.1.988__py3-none-any.whl → 1.1.997__py3-none-any.whl

biolib/biolib_binary_format/remote_endpoints.py

@@ -1,6 +1,8 @@
 import datetime
 from urllib.parse import urlparse, parse_qs
 
+from biolib.biolib_logging import logger
+
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format.utils import RemoteEndpoint
 
@@ -23,5 +25,6 @@ class RemoteJobStorageResultEndpoint(RemoteEndpoint):
             query_params = parse_qs(parsed_url.query)
             time_at_generation = datetime.datetime.strptime(query_params['X-Amz-Date'][0], '%Y%m%dT%H%M%SZ')
             self._expires_at = time_at_generation + datetime.timedelta(seconds=int(query_params['X-Amz-Expires'][0]))
+            logger.debug(f'Job "{self._job_id}" fetched presigned URL with expiry at {self._expires_at.isoformat()}')
 
         return self.presigned_url

biolib/biolib_binary_format/utils.py

@@ -1,10 +1,12 @@
 from abc import ABC, abstractmethod
-
 import io
+import time
 
 import requests
+from requests import Response
 
-from biolib.typing_utils import Iterable
+from biolib.biolib_logging import logger
+from biolib.typing_utils import Iterable, Optional, Dict
 from biolib.utils.multipart_uploader import get_chunk_iterator_from_bytes
 
 
@@ -49,6 +51,46 @@ class IndexableBuffer(ABC):
         return data
 
 
+def _get_request_with_retries_for_timeout(
+        url: str,
+        headers: Optional[Dict[str, str]] = None,
+        stream: bool = False,
+) -> Response:
+    retries = 10
+    for retry_count in range(retries):
+        if retry_count > 0:
+            time.sleep(1 * retry_count)
+            logger.debug('Retrying HTTP GET request...')
+
+        try:
+            response = requests.get(
+                headers=headers,
+                stream=stream,
+                timeout=10,
+                url=url,
+            )
+        except requests.exceptions.ReadTimeout:
+            logger.debug('Got read timeout of HTTP GET request')
+            continue
+
+        if response.status_code == 502:
+            logger.debug('Got 502 status for HTTP GET')
+            continue
+
+        if response.status_code == 400:
+            logger.error(
+                f'GET request for "{url}" got response with status {response.status_code} and '
+                f'content: {response.content.decode()}'
+            )
+
+        if not response.ok:
+            raise Exception(f'GET request for "{url}" got response with status {response.status_code}')
+
+        return response
+
+    raise Exception(f'Failed to complete HTTP GET request for "{url}" after {retries} retries')
+
+
 class RemoteIndexableBuffer(IndexableBuffer):
 
     def __init__(self, endpoint: RemoteEndpoint):
@@ -63,9 +105,10 @@ class RemoteIndexableBuffer(IndexableBuffer):
             return bytes(0)
 
         end = start + length - 1
-        response = requests.get(url=self._endpoint.get_remote_url(), headers={'range': f'bytes={start}-{end}'})
-        if not response.ok:
-            raise Exception(f'get_data got not ok response status {response.status_code}')
+        response = _get_request_with_retries_for_timeout(
+            url=self._endpoint.get_remote_url(),
+            headers={'range': f'bytes={start}-{end}'},
+        )
 
         data: bytes = response.content
         if len(data) != length:
@@ -81,11 +124,10 @@ class RemoteIndexableBuffer(IndexableBuffer):
             return []
 
         end = start + length - 1
-        response = requests.get(
+        response = _get_request_with_retries_for_timeout(
             url=self._endpoint.get_remote_url(),
             headers={'range': f'bytes={start}-{end}'},
             stream=True,
-            timeout=60,
         )
         return response.iter_content(chunk_size=chunk_size)
 

{pybiolib-1.1.988.dist-info → pybiolib-1.1.997.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.988
+Version: 1.1.997
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.988.dist-info → pybiolib-1.1.997.dist-info}/RECORD

@@ -25,14 +25,14 @@ biolib/biolib_binary_format/common_types.py,sha256=wlxstqO0Wajw8y_Rt6Gmiw4dZlf2D
 biolib/biolib_binary_format/encrypted_module_output.py,sha256=yqqfhnNymVd22cazn1Y8gDganLKiAHB2qptr4MP9BmU,6450
 biolib/biolib_binary_format/module_input.py,sha256=E-mvq26NFpYM7fqMMj8Nrpqxezy4CWRA1OS3hJK1kk0,2603
 biolib/biolib_binary_format/module_output.py,sha256=8tExFYF2kBXK3VyIRMptrRxpBN-ImxUR3PJmk6wK5jA,2640
-biolib/biolib_binary_format/remote_endpoints.py,sha256=k9PCHsFtMwcF1F3lWmeE88vY-1ejQn35ntRsNibmYV4,1146
+biolib/biolib_binary_format/remote_endpoints.py,sha256=LgWd_cewZK4ncPR8303GM_PmIEZZJ5RQatJ_NSXUgbU,1306
 biolib/biolib_binary_format/rsa_encrypted_aes_package.py,sha256=p2IAv1_FH67qQV3rj1fXKDQNSFxMuLaHEbuJ6WxRLdc,2269
 biolib/biolib_binary_format/saved_job.py,sha256=nFHVFRNTNcAFGODLSiBntCtMk55QKwreUq6qLX80dI4,1125
 biolib/biolib_binary_format/stdout_and_stderr.py,sha256=WfUUJFFCBrtfXjuWIaRPiWCpuBLxfko68oxoTKhrwx8,1023
 biolib/biolib_binary_format/system_exception.py,sha256=T3iL4_cSHAHim3RSDPS8Xyb1mfteaJBZonSXuRltc28,853
 biolib/biolib_binary_format/system_status_update.py,sha256=aOELuQ0k-GtpaZTUxYd0GFomP_OInmrK585y6fuQuKE,1191
 biolib/biolib_binary_format/unencrypted_module_output.py,sha256=Op9NqlSwwoRKbf7ecHgBChrXdJHEAzKjREojv6d5I3g,8514
-biolib/biolib_binary_format/utils.py,sha256=n96tiAQhDi_rZNA4uAvnbo2m8NyriJNhwpQAMLOy25E,4101
+biolib/biolib_binary_format/utils.py,sha256=wAC21bfqvz27Ug41kKaASouYChXNYh49DEI8_VKhWfk,5402
 biolib/biolib_docker_client/__init__.py,sha256=aBfA6mtWSI5dBEfNNMD6bIZzCPloW4ghKm0wqQiljdo,1481
 biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
 biolib/biolib_logging.py,sha256=J3E5H_LL5k6ZUim2C8gqN7E6lCBZMTpO4tnMpOPwG9U,2854
@@ -107,8 +107,8 @@ biolib/validators/validate_argument.py,sha256=B-tqgMgXuTvJLhaMWcBTUKw18L5brJtIdf
 biolib/validators/validate_module.py,sha256=aHe0YAWFSrMbZNw-h64oweqruDS-k7KOUzK1ECpG8Wg,13298
 biolib/validators/validate_zip_file.py,sha256=2j8kJ1WF2u5OyMkfEDCoU_BvOtr8gLPWTtsmUG0zQ5g,1655
 biolib/validators/validator_utils.py,sha256=2QCJWdKiLTUHsvjDidrXKBjH95kKsEmFwd17muTGL0M,2135
-pybiolib-1.1.988.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.988.dist-info/METADATA,sha256=qTgiAy3kZ2RNdm234TGjUK1_AFci5-j996O0XpERXcM,1611
-pybiolib-1.1.988.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-pybiolib-1.1.988.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.988.dist-info/RECORD,,
+pybiolib-1.1.997.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.997.dist-info/METADATA,sha256=YrBf4WneR9rWEcM86GPgz_JEByZyelZHBAiL_EtkMN8,1611
+pybiolib-1.1.997.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+pybiolib-1.1.997.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.997.dist-info/RECORD,,