pybiolib 1.1.1718__tar.gz → 1.1.1730__tar.gz

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1718
+Version: 1.1.1730
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/_internal/http_client.py

@@ -43,9 +43,10 @@ class HttpError(urllib.error.HTTPError):
 
 class HttpResponse:
     def __init__(self, response):
-        self.status_code = response.status
-        self.content = response.read()
-        self.url = response.geturl()
+        self.headers: Dict[str, str] = dict(response.headers)
+        self.status_code: int = int(response.status)
+        self.content: bytes = response.read()
+        self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
@@ -61,10 +62,11 @@ class HttpClient:
     @staticmethod
     def request(
             url: str,
-            method: Optional[Literal['GET', 'POST', 'PATCH']] = None,
+            method: Optional[Literal['GET', 'POST', 'PATCH', 'PUT']] = None,
             data: Optional[Union[Dict, bytes]] = None,
             headers: Optional[Dict[str, str]] = None,
             retries: int = 5,
+            timeout_in_seconds: Optional[int] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
             HttpClient.ssl_context = _create_ssl_context()
@@ -79,7 +81,8 @@ class HttpClient:
             headers=headers_to_send,
             method=method or 'GET',
         )
-        timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
+        if timeout_in_seconds is None:
+            timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
 
         last_error: Optional[urllib.error.URLError] = None
         for retry_count in range(retries + 1):
@@ -90,7 +93,7 @@ class HttpClient:
                 with urllib.request.urlopen(
                     request,
                     context=HttpClient.ssl_context,
-                    timeout=timeout_in_seconds
+                    timeout=timeout_in_seconds,
                 ) as response:
                     return HttpResponse(response)
 

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/api/client.py

@@ -4,7 +4,10 @@ from biolib.typing_utils import Dict, Optional, Union
 from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
 from biolib._internal.http_client import HttpResponse, HttpClient
 
-OptionalHeaders = Optional[Dict[str, Union[str, None]]]
+OptionalHeaders = Union[
+    Optional[Dict[str, str]],
+    Optional[Dict[str, Union[str, None]]],
+]
 
 
 class ApiClient(HttpClient):

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/compute_node/job_worker/job_storage.py

@@ -2,7 +2,7 @@ import os
 
 from biolib import utils
 from biolib._internal.http_client import HttpClient
-from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
+from biolib.biolib_api_client import CreatedJobDict
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.biolib_logging import logger_no_user_data
@@ -14,24 +14,23 @@ class JobStorage:
 
     @staticmethod
     def upload_module_input(job: CreatedJobDict, module_input_serialized: bytes) -> None:
-        base_url = BiolibApiClient.get().base_url
         job_uuid = job['public_id']
         headers = {'Job-Auth-Token': job['auth_token']}
 
         multipart_uploader = utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/start_upload/',
                 headers=headers
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/input/presigned_upload_url/',
                 headers=headers
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/complete_upload/',
                 headers=headers
             ),
         )
@@ -59,7 +58,6 @@ class JobStorage:
 
     @staticmethod
     def _get_module_output_uploader(job_uuid: str) -> utils.MultiPartUploader:
-        base_url = BiolibApiClient.get().base_url
         config = CloudUtils.get_webserver_config()
         compute_node_auth_token = config['compute_node_info']['auth_token']  # pylint: disable=unsubscriptable-object
         headers = {'Compute-Node-Auth-Token': compute_node_auth_token}
@@ -67,17 +65,17 @@ class JobStorage:
         return utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/start_upload/',
                 headers=headers,
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/results/presigned_upload_url/',
                 headers=headers,
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/complete_upload/',
                 headers=headers,
             ),
         )

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/lfs/utils.py

@@ -6,9 +6,8 @@ from collections import namedtuple
 from pathlib import Path
 from struct import Struct
 
-import requests
-
 from biolib import utils
+from biolib._internal.http_client import HttpClient
 from biolib.app import BioLibApp
 from biolib.biolib_api_client.biolib_account_api import BiolibAccountApi
 from biolib.biolib_api_client.biolib_large_file_system_api import BiolibLargeFileSystemApi
@@ -143,18 +142,17 @@ def push_large_file_system(lfs_uri: str, input_dir: str, chunk_size_in_mb: Optio
 
     iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
 
-    base_url = BiolibApiClient.get().base_url
     multipart_uploader = utils.MultiPartUploader(
         use_process_pool=True,
         get_presigned_upload_url_request=dict(
             headers=None,
             requires_biolib_auth=True,
-            url=f'{base_url}/api/lfs/versions/{lfs_resource_version_uuid}/presigned_upload_url/',
+            path=f'/lfs/versions/{lfs_resource_version_uuid}/presigned_upload_url/',
         ),
         complete_upload_request=dict(
             headers=None,
             requires_biolib_auth=True,
-            url=f'{base_url}/api/lfs/versions/{lfs_resource_version_uuid}/complete_upload/',
+            path=f'/lfs/versions/{lfs_resource_version_uuid}/complete_upload/',
         ),
     )
 
@@ -221,25 +219,19 @@ def get_file_data_from_large_file_system(lfs_uri: str, file_path: str) -> bytes:
         local_file_header_start = file_info['header_offset'] + len(local_file_header_signature_bytes)
         local_file_header_end = local_file_header_start + local_file_header_struct.size
 
-        local_file_header_response = requests.get(
+        local_file_header_response = HttpClient.request(
             url=lfs_url,
-            stream=True,
             headers={'range': f'bytes={local_file_header_start}-{local_file_header_end - 1}'},
-            timeout=300,
+            timeout_in_seconds=300,
         )
-        local_file_header_response.raise_for_status()
-        local_file_header_bytes: bytes = local_file_header_response.raw.data
-        local_file_header = LocalFileHeader._make(local_file_header_struct.unpack(local_file_header_bytes))
+        local_file_header = LocalFileHeader._make(local_file_header_struct.unpack(local_file_header_response.content))
 
         file_start = local_file_header_end + local_file_header.file_name_len + local_file_header.extra_field_len
         file_end = file_start + file_info['file_size']
 
-        response = requests.get(
+        response = HttpClient.request(
             url=lfs_url,
-            stream=True,
             headers={'range': f'bytes={file_start}-{file_end - 1}'},
-            timeout=300,  # timeout after 5 min
+            timeout_in_seconds=300,  # timeout after 5 min
         )
-        response.raise_for_status()
-        data: bytes = response.raw.data
-        return data
+        return response.content

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/utils/__init__.py

@@ -1,22 +1,19 @@
 import collections.abc
 import multiprocessing
 import os
-import time
 import socket
 import sys
 
 from typing import Optional
-import requests
 from importlib_metadata import version, PackageNotFoundError
 
 from biolib.utils.seq_util import SeqUtil, SeqUtilRecord
-
-# try fetching version, if it fails (usually when in dev), add default
-from biolib.biolib_errors import BioLibError
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_logging import logger_no_user_data, logger
 from biolib.typing_utils import Tuple, Iterator
 from .multipart_uploader import MultiPartUploader, get_chunk_iterator_from_bytes
 
+# try fetching version, if it fails (usually when in dev), add default
 try:
     BIOLIB_PACKAGE_VERSION = version('pybiolib')
 except PackageNotFoundError:
@@ -87,38 +84,17 @@ DownloadChunkInputTuple = Tuple[ByteRangeTuple, str]
 
 
 def _download_chunk(input_tuple: DownloadChunkInputTuple) -> bytes:
-    max_download_retries = 10
-
     byte_range, presigned_url = input_tuple
     start, end = byte_range
 
-    for retry_attempt in range(max_download_retries):
-        if retry_attempt > 0:
-            logger_no_user_data.debug(f'Attempt number {retry_attempt} for part {start}')
-        try:
-            response = requests.get(
-                url=presigned_url,
-                stream=True,
-                headers={'range': f'bytes={start}-{end}'},
-                timeout=300,  # timeout after 5 min
-            )
-            if response.ok:
-                return_value: bytes = response.raw.data
-                logger_no_user_data.debug(f'Returning raw data for part {start}')
-                return return_value
-            else:
-                logger_no_user_data.warning(
-                    f'Got not ok response when downloading part {start}:{end}. '
-                    f'Got response status {response.status_code} and content: {response.content.decode()} '
-                    f'Retrying...'
-                )
-        except Exception:  # pylint: disable=broad-except
-            logger_no_user_data.warning(f'Encountered error when downloading part {start}:{end}. Retrying...')
-
-        time.sleep(5)
-
-    logger_no_user_data.debug(f'Max retries hit, when downloading part {start}:{end}. Exiting...')
-    raise BioLibError(f'Max retries hit, when downloading part {start}:{end}. Exiting...')
+    response = HttpClient.request(
+        url=presigned_url,
+        headers={'range': f'bytes={start}-{end}'},
+        timeout_in_seconds=300,  # timeout after 5 min
+        retries=10,
+    )
+    logger_no_user_data.debug(f'Returning raw data for part {start}')
+    return response.content
 
 
 class ChunkIterator(collections.abc.Iterator):
@@ -153,11 +129,8 @@ class ChunkIterator(collections.abc.Iterator):
 def download_presigned_s3_url(presigned_url: str, output_file_path: str) -> None:
     chunk_size = 50_000_000
 
-    with requests.get(presigned_url, stream=True, headers={'range': 'bytes=0-1'}) as response:
-        if not response.ok:
-            raise Exception(f'Got response status code {response.status_code} and content {response.content.decode()}')
-
-        file_size = int(response.headers['Content-Range'].split('/')[1])
+    response = HttpClient.request(url=presigned_url, headers={'range': 'bytes=0-1'})
+    file_size = int(response.headers['Content-Range'].split('/')[1])
 
     chunk_iterator = ChunkIterator(file_size, chunk_size, presigned_url)
 

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/biolib/utils/multipart_uploader.py

@@ -5,10 +5,9 @@ import os
 import time
 from urllib.parse import urlparse
 
-import requests
-
+import biolib.api
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.auth import BearerAuth
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import TypedDict, List, Iterator, Tuple, Optional, Dict
@@ -33,7 +32,7 @@ def get_chunk_iterator_from_file_object(file_object, chunk_size_in_bytes: int =
 class RequestOptions(TypedDict):
     headers: Optional[Dict[str, str]]
     requires_biolib_auth: bool
-    url: str
+    path: str
 
 
 class _PartMetadata(TypedDict):
@@ -67,20 +66,15 @@ class MultiPartUploader:
         logger_no_user_data.debug(f'Starting multipart upload of payload with size {payload_size_in_bytes} bytes')
 
         if self._start_multipart_upload_request:
-            requires_biolib_auth = self._start_multipart_upload_request['requires_biolib_auth']
-            start_multipart_upload = requests.post(
-                auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
-                headers=self._start_multipart_upload_request['headers'],
-                timeout=30,
-                url=self._start_multipart_upload_request['url'],
-            )
-            if start_multipart_upload.ok:
-                logger_no_user_data.debug('Multipart upload started')
-            else:
-                logger_no_user_data.debug(
-                    f'Failed to start multipart upload got response status: {start_multipart_upload.status_code}'
+            try:
+                biolib.api.client.post(
+                    authenticate=self._start_multipart_upload_request['requires_biolib_auth'],
+                    headers=self._start_multipart_upload_request['headers'],
+                    path=self._start_multipart_upload_request['path'],
                 )
-                raise Exception('Failed to start multipart upload')
+            except BaseException as error:
+                logger_no_user_data.debug(f'Failed to start multipart upload got error: {error}')
+                raise error
 
         # if multiprocessing start method is spawn or we are running in a daemon process,
         # multiprocessing.Pool may fail when called from script
@@ -116,30 +110,12 @@ class MultiPartUploader:
             BiolibApiClient.refresh_auth_token()
 
         logger_no_user_data.debug(f'Uploaded {len(parts)} parts, now calling complete upload...')
-        for index in range(3):
-            try:
-                complete_upload_response = requests.post(
-                    auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
-                    headers=self._complete_upload_request['headers'],
-                    json={'parts': parts, 'size_bytes': self._bytes_uploaded},
-                    timeout=30,
-                    url=self._complete_upload_request['url'],
-                )
-                if complete_upload_response.ok:
-                    logger_no_user_data.debug('Multipart upload completed returning')
-                    return
-
-                logger_no_user_data.warning(
-                    f'Failed to complete multipart upload got response status {complete_upload_response.status_code}. '
-                    f'Retrying...'
-                )
-
-            except Exception as error:  # pylint: disable=broad-except
-                logger_no_user_data.warning('Encountered error when completing multipart upload. Retrying...')
-                logger.debug(f'Multipart complete error: {error}')
-                time.sleep(index * index + 2)
-
-        raise BioLibError('Max retries hit, when completing multipart upload')
+        biolib.api.client.post(
+            authenticate=requires_biolib_auth,
+            headers=self._complete_upload_request['headers'],
+            data={'parts': parts, 'size_bytes': self._bytes_uploaded},
+            path=self._complete_upload_request['path'],
+        )
 
     def _upload_chunk(self, _input: _UploadChunkInputType) -> _UploadChunkReturnType:
         part_number, chunk = _input
@@ -152,18 +128,12 @@ class MultiPartUploader:
             logger_no_user_data.debug(f'Uploading part number {part_number} with size {len(chunk)} bytes...')
             try:
                 logger_no_user_data.debug(f'Getting upload URL for chunk {part_number}...')
-                get_url_response = requests.get(
-                    auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
+                get_url_response = biolib.api.client.get(
+                    authenticate=requires_biolib_auth,
                     headers=self._get_presigned_upload_url_request['headers'],
                     params={'part_number': part_number},
-                    timeout=30,
-                    url=self._get_presigned_upload_url_request['url'],
+                    path=self._get_presigned_upload_url_request['path'],
                 )
-                if not get_url_response.ok:
-                    raise Exception(
-                        f'Failed to get upload URL for part {part_number} got response status code '
-                        f'{get_url_response.status_code}'
-                    )
 
                 presigned_upload_url = get_url_response.json()['presigned_upload_url']
 
@@ -174,16 +144,13 @@ class MultiPartUploader:
                     presigned_upload_url = \
                         f'{app_caller_proxy_job_storage_base_url}{parsed_url.path}?{parsed_url.query}'
 
-                put_chunk_response = requests.put(url=presigned_upload_url, data=chunk, timeout=300)
-
-                if put_chunk_response.ok:
-                    return _PartMetadata(PartNumber=part_number, ETag=put_chunk_response.headers['ETag']), len(chunk)
-                else:
-                    logger_no_user_data.warning(
-                        f'Got response with status {put_chunk_response.status_code} when uploading part {part_number}. '
-                        'Retrying...'
-                    )
-                    logger.debug(f'Response content: {put_chunk_response.content.decode()}')
+                put_chunk_response = HttpClient.request(
+                    url=presigned_upload_url,
+                    data=chunk,
+                    method='PUT',
+                    timeout_in_seconds=300,
+                )
+                return _PartMetadata(PartNumber=part_number, ETag=put_chunk_response.headers['ETag']), len(chunk)
 
             except Exception as error:  # pylint: disable=broad-except
                 logger_no_user_data.warning(f'Encountered error when uploading part {part_number}. Retrying...')

{pybiolib-1.1.1718 → pybiolib-1.1.1730}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.1.1718"
+version = "1.1.1730"
 description = "BioLib Python Client"
 readme = "README.md"
 license = "MIT"