pybiolib 1.1.1711__tar.gz → 1.1.1730__tar.gz

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1711
+Version: 1.1.1730
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/_internal/http_client.py

@@ -11,7 +11,8 @@ import urllib.parse
 from biolib.biolib_logging import logger_no_user_data
 from biolib.typing_utils import Dict, Optional, Union, Literal, cast
 
-def create_ssl_context():
+
+def _create_ssl_context():
     context = ssl.create_default_context()
     try:
         if platform.system() == 'Darwin':
@@ -42,9 +43,10 @@ class HttpError(urllib.error.HTTPError):
 
 class HttpResponse:
     def __init__(self, response):
-        self.status_code = response.status
-        self.content = response.read()
-        self.url = response.geturl()
+        self.headers: Dict[str, str] = dict(response.headers)
+        self.status_code: int = int(response.status)
+        self.content: bytes = response.read()
+        self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
@@ -60,13 +62,14 @@ class HttpClient:
     @staticmethod
     def request(
             url: str,
-            method: Optional[Literal['GET', 'POST', 'PATCH']] = None,
+            method: Optional[Literal['GET', 'POST', 'PATCH', 'PUT']] = None,
             data: Optional[Union[Dict, bytes]] = None,
             headers: Optional[Dict[str, str]] = None,
             retries: int = 5,
+            timeout_in_seconds: Optional[int] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
-            HttpClient.ssl_context = create_ssl_context()
+            HttpClient.ssl_context = _create_ssl_context()
         headers_to_send = headers or {}
         if isinstance(data, dict):
             headers_to_send['Accept'] = 'application/json'
@@ -78,7 +81,8 @@ class HttpClient:
             headers=headers_to_send,
             method=method or 'GET',
         )
-        timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
+        if timeout_in_seconds is None:
+            timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
 
         last_error: Optional[urllib.error.URLError] = None
         for retry_count in range(retries + 1):
@@ -89,7 +93,7 @@ class HttpClient:
                 with urllib.request.urlopen(
                     request,
                     context=HttpClient.ssl_context,
-                    timeout=timeout_in_seconds
+                    timeout=timeout_in_seconds,
                 ) as response:
                     return HttpResponse(response)
 

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/api/client.py

@@ -4,7 +4,10 @@ from biolib.typing_utils import Dict, Optional, Union
 from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
 from biolib._internal.http_client import HttpResponse, HttpClient
 
-OptionalHeaders = Optional[Dict[str, Union[str, None]]]
+OptionalHeaders = Union[
+    Optional[Dict[str, str]],
+    Optional[Dict[str, Union[str, None]]],
+]
 
 
 class ApiClient(HttpClient):
@@ -14,11 +17,12 @@ class ApiClient(HttpClient):
             params: Optional[Dict[str, Union[str, int]]] = None,
             headers: OptionalHeaders = None,
             authenticate: bool = True,
+            retries: int = 10,
     ) -> HttpResponse:
         return self.request(
             headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
             method='GET',
-            retries=10,
+            retries=retries,
             url=self._get_absolute_url(path=path, query_params=params),
         )
 
@@ -27,21 +31,23 @@ class ApiClient(HttpClient):
             path: str,
             data: Optional[Union[Dict, bytes]] = None,
             headers: OptionalHeaders = None,
+            authenticate: bool = True,
+            retries: int = 5,
     ) -> HttpResponse:
         return self.request(
             data=data,
-            headers=self._get_headers(opt_headers=headers, authenticate=True),
+            headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
             method='POST',
-            retries=5,
+            retries=retries,
             url=self._get_absolute_url(path=path, query_params=None),
         )
 
-    def patch(self, path: str, data: Dict, headers: OptionalHeaders = None) -> HttpResponse:
+    def patch(self, path: str, data: Dict, headers: OptionalHeaders = None, retries: int = 5) -> HttpResponse:
         return self.request(
             data=data,
             headers=self._get_headers(opt_headers=headers, authenticate=True),
             method='PATCH',
-            retries=5,
+            retries=retries,
             url=self._get_absolute_url(path=path, query_params=None),
         )
 

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/biolib_api_client/biolib_job_api.py

@@ -1,16 +1,12 @@
 import os
-import time
 from urllib.parse import urlparse
 
-import requests
-
 import biolib.api
 
 from biolib import utils
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient, CloudJob, JobState
-from biolib.biolib_errors import BioLibError, RetryLimitException, StorageDownloadFailed, JobResultPermissionError, \
-    JobResultError, JobResultNotFound
+from biolib._internal.http_client import HttpError
+from biolib.biolib_api_client import CloudJob, JobState
+from biolib.biolib_errors import JobResultPermissionError, JobResultError, JobResultNotFound, StorageDownloadFailed
 from biolib.biolib_logging import logger
 from biolib.utils import BIOLIB_PACKAGE_VERSION
 from biolib.typing_utils import TypedDict, Optional, Literal, Dict
@@ -96,35 +92,11 @@ class BiolibJobApi:
 
     @staticmethod
     def create_cloud_job(job_id: str, result_name_prefix: Optional[str]) -> CloudJob:
-        response = None
         data = {'job_id': job_id}
         if result_name_prefix:
             data['result_name_prefix'] = result_name_prefix
 
-        for retry in range(4):
-            try:
-                response = requests.post(
-                    f'{BiolibApiClient.get().base_url}/api/jobs/cloud/',
-                    json=data,
-                    auth=BearerAuth(BiolibApiClient.get().access_token)
-                )
-
-                if response.status_code == 503:
-                    raise RetryLimitException(response.content)
-                # Handle possible validation errors from backend
-                elif not response.ok:
-                    raise BioLibError(response.text)
-
-                break
-
-            except RetryLimitException as retry_exception:  # pylint: disable=broad-except
-                if retry > 3:
-                    raise BioLibError('Reached retry limit for cloud job creation') from retry_exception
-                time.sleep(1)
-
-        if not response:
-            raise BioLibError('Could not create new cloud job')
-
+        response = biolib.api.client.post(path='/jobs/cloud/', data=data)
         cloud_job: CloudJob = response.json()
         return cloud_job
 
@@ -136,9 +108,9 @@ class BiolibJobApi:
     ) -> str:
         try:
             response = biolib.api.client.get(
-                path=f'{BiolibApiClient.get().base_url}/api/jobs/{job_uuid}/storage/{storage_type}/download/',
+                path=f'/jobs/{job_uuid}/storage/{storage_type}/download/',
                 authenticate=True,
-                headers={'Job-Auth-Token': job_auth_token}
+                headers={'Job-Auth-Token': job_auth_token},
             )
             presigned_s3_download_link_response: PresignedS3DownloadLinkResponse = response.json()
             presigned_download_url = presigned_s3_download_link_response['presigned_download_url']
@@ -151,21 +123,20 @@ class BiolibJobApi:
 
             return presigned_download_url
 
-        except requests.exceptions.HTTPError as error:
-            status_code = error.response.status_code
+        except HttpError as error:
             if storage_type == 'results':
-                if status_code == 401:
+                if error.code == 401:
                     raise JobResultPermissionError('You must be signed in to get result of the job') from None
-                elif status_code == 403:
+                elif error.code == 403:
                     raise JobResultPermissionError(
                         'Cannot get result of job. Maybe the job was created without being signed in?'
                     ) from None
-                elif status_code == 404:
+                elif error.code == 404:
                     raise JobResultNotFound('Job result not found') from None
                 else:
                     raise JobResultError('Failed to get result of job') from error
             else:
-                raise StorageDownloadFailed(error.response.content) from error
+                raise StorageDownloadFailed(f'Failed to download result of job got error: {error}') from error
 
         except Exception as error:  # pylint: disable=broad-except
             if storage_type == 'results':

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/compute_node/cloud_utils/cloud_utils.py

@@ -6,12 +6,10 @@ import subprocess
 import time
 from datetime import datetime
 from socket import gethostbyname, gethostname
-import requests
 
-from biolib import utils
-from biolib.biolib_errors import BioLibError
+from biolib import utils, api
 from biolib.biolib_logging import logger_no_user_data
-from biolib.typing_utils import Optional, List
+from biolib.typing_utils import Optional, List, Dict, cast
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.compute_node.webserver.webserver_types import WebserverConfig, ComputeNodeInfo, ShutdownTimes
 
@@ -74,18 +72,15 @@ class CloudUtils:
         if utils.IS_RUNNING_IN_CLOUD:
             config = CloudUtils.get_webserver_config()
             try:
-                response = requests.post(url=f'{config["base_url"]}/api/jobs/deregister/', json={
-                    'auth_token': config["compute_node_info"]["auth_token"],
-                    'public_id': config["compute_node_info"]["public_id"],
-                })
-
-                if not response.ok:
-                    response_content = response.content.decode()
-                    logger_no_user_data.error(
-                        f'Failed to deregister got status {response.status_code} and response {response_content}'
-                    )
-
-            except Exception as error:  # pylint: disable=broad-except
+                api.client.post(
+                    authenticate=False,
+                    path='/jobs/deregister/',
+                    data={
+                        'auth_token': config['compute_node_info']['auth_token'],
+                        'public_id': config['compute_node_info']['public_id'],
+                    },
+                )
+            except BaseException as error:
                 logger_no_user_data.error(f'Failed to deregister got error: {error}')
         else:
             logger_no_user_data.error("Not deregistering as environment is not cloud")
@@ -120,34 +115,26 @@ class CloudUtils:
             )
             return
 
-        config = CloudUtils.get_webserver_config()
-
-        for _ in range(100):
-            try:
-                response = requests.post(
-                    url=f'{config["base_url"]}/api/jobs/cloud/finish/',
-                    json={
-                        'auth_token': config["compute_node_info"]["auth_token"],
-                        'cloud_job_id': cloud_job_id,
-                        'system_exception_code': system_exception_code,
-                        'exit_code': exit_code
-                    },
-                    timeout=10,
-                )
-
-                response.raise_for_status()
-
-                opt_error_string = f' with error code {system_exception_code}' if system_exception_code else ''
-                logger_no_user_data.debug(
-                    f'Cloud Job "{cloud_job_id}" was reported as finished' + opt_error_string
-                )
-                return
+        logger_no_user_data.debug(
+            f'Reporting CloudJob "{cloud_job_id}" as finished with exit code: {exit_code} '
+            f'and system exception code: {system_exception_code}'
+        )
 
-            except Exception as error:  # pylint: disable=broad-except
-                logger_no_user_data.debug(
-                    f'Could not finish cloud job "{cloud_job_id}" due to {error}, retrying...'
-                )
-                time.sleep(10)
+        config = CloudUtils.get_webserver_config()
+        try:
+            api.client.post(
+                authenticate=False,
+                path='/jobs/cloud/finish/',
+                retries=100,
+                data={
+                    'auth_token': config['compute_node_info']['auth_token'],
+                    'cloud_job_id': cloud_job_id,
+                    'system_exception_code': system_exception_code,
+                    'exit_code': exit_code
+                },
+            )
+        except BaseException as error:
+            logger_no_user_data.debug(f'Failed to finish CloudJob "{cloud_job_id}" due to: {error}')
 
     @staticmethod
     def _report_availability() -> None:
@@ -158,26 +145,11 @@ class CloudUtils:
             logger_no_user_data.debug(
                 f'Registering with {compute_node_info} to host {api_client.base_url} at {datetime.now()}'
             )
-
-            response: Optional[requests.Response] = None
-            max_retries = 5
-            for retry_count in range(max_retries):
-                try:
-                    response = requests.post(
-                        url=f'{api_client.base_url}/api/jobs/report_available/',
-                        json=compute_node_info,
-                    )
-                    break
-                except Exception as error:  # pylint: disable=broad-except
-                    logger_no_user_data.error(f'Self-registering failed with error: {error}')
-                    if retry_count < max_retries - 1:
-                        seconds_to_sleep = 1
-                        logger_no_user_data.info(f'Retrying self-registering in {seconds_to_sleep} seconds')
-                        time.sleep(seconds_to_sleep)
-
-            if not response:
-                raise BioLibError('Failed to register. Max retry limit reached')
-
+            response = api.client.post(
+                authenticate=False,
+                path='/jobs/report_available/',
+                data=cast(Dict[str, str], compute_node_info),
+            )
             if response.status_code != 201:
                 raise Exception("Non 201 error code")
             else:

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/compute_node/job_worker/job_storage.py

@@ -1,9 +1,8 @@
 import os
 
-import requests
-
 from biolib import utils
-from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
+from biolib._internal.http_client import HttpClient
+from biolib.biolib_api_client import CreatedJobDict
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.biolib_logging import logger_no_user_data
@@ -15,24 +14,23 @@ class JobStorage:
 
     @staticmethod
     def upload_module_input(job: CreatedJobDict, module_input_serialized: bytes) -> None:
-        base_url = BiolibApiClient.get().base_url
         job_uuid = job['public_id']
         headers = {'Job-Auth-Token': job['auth_token']}
 
         multipart_uploader = utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/start_upload/',
                 headers=headers
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/input/presigned_upload_url/',
                 headers=headers
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/complete_upload/',
                 headers=headers
             ),
         )
@@ -60,7 +58,6 @@ class JobStorage:
 
     @staticmethod
     def _get_module_output_uploader(job_uuid: str) -> utils.MultiPartUploader:
-        base_url = BiolibApiClient.get().base_url
         config = CloudUtils.get_webserver_config()
         compute_node_auth_token = config['compute_node_info']['auth_token']  # pylint: disable=unsubscriptable-object
         headers = {'Compute-Node-Auth-Token': compute_node_auth_token}
@@ -68,17 +65,17 @@ class JobStorage:
         return utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/start_upload/',
                 headers=headers,
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/results/presigned_upload_url/',
                 headers=headers,
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/complete_upload/',
                 headers=headers,
             ),
         )
@@ -92,8 +89,7 @@ class JobStorage:
             job_auth_token=job['auth_token'],
             storage_type='input',
         )
-        response = requests.get(url=presigned_download_url)
-        response.raise_for_status()
+        response = HttpClient.request(url=presigned_download_url)
         data: bytes = response.content
         logger_no_user_data.debug(f'Job "{job_uuid}" module input downloaded')
         return data

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/compute_node/job_worker/job_worker.py

@@ -12,9 +12,9 @@ import os
 import signal
 from types import FrameType
 
-import requests
 from docker.models.networks import Network  # type: ignore
 
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
 from biolib.compute_node.job_worker.job_legacy_input_wait_timeout_thread import JobLegacyInputWaitTimeout
 from biolib.compute_node.job_worker.job_storage import JobStorage
@@ -526,7 +526,7 @@ class JobWorker:
             start_time = time()
             logger_no_user_data.debug(f'Job "{root_job_id}" downloading runtime zip...')
             try:
-                runtime_zip_bytes = requests.get(runtime_zip_url).content
+                runtime_zip_bytes = HttpClient.request(url=runtime_zip_url).content
             except Exception as exception:
                 raise ComputeProcessException(
                     exception,

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/jobs/job.py

@@ -6,7 +6,6 @@ from pathlib import Path
 from collections import OrderedDict
 from urllib.parse import urlparse
 
-import requests
 from biolib import api, utils
 from biolib._internal.http_client import HttpClient
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
@@ -127,8 +126,7 @@ class Job:
             job_auth_token=self._job_dict['auth_token'],
             storage_type='input',
         )
-        response = requests.get(url=presigned_download_url)
-        response.raise_for_status()
+        response = HttpClient.request(url=presigned_download_url)
         module_input_serialized: bytes = response.content
         return ModuleInput(module_input_serialized).deserialize()
 

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/lfs/utils.py

@@ -6,9 +6,8 @@ from collections import namedtuple
 from pathlib import Path
 from struct import Struct
 
-import requests
-
 from biolib import utils
+from biolib._internal.http_client import HttpClient
 from biolib.app import BioLibApp
 from biolib.biolib_api_client.biolib_account_api import BiolibAccountApi
 from biolib.biolib_api_client.biolib_large_file_system_api import BiolibLargeFileSystemApi
@@ -143,18 +142,17 @@ def push_large_file_system(lfs_uri: str, input_dir: str, chunk_size_in_mb: Optio
 
     iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
 
-    base_url = BiolibApiClient.get().base_url
     multipart_uploader = utils.MultiPartUploader(
         use_process_pool=True,
         get_presigned_upload_url_request=dict(
             headers=None,
             requires_biolib_auth=True,
-            url=f'{base_url}/api/lfs/versions/{lfs_resource_version_uuid}/presigned_upload_url/',
+            path=f'/lfs/versions/{lfs_resource_version_uuid}/presigned_upload_url/',
         ),
         complete_upload_request=dict(
             headers=None,
             requires_biolib_auth=True,
-            url=f'{base_url}/api/lfs/versions/{lfs_resource_version_uuid}/complete_upload/',
+            path=f'/lfs/versions/{lfs_resource_version_uuid}/complete_upload/',
         ),
     )
 
@@ -221,25 +219,19 @@ def get_file_data_from_large_file_system(lfs_uri: str, file_path: str) -> bytes:
         local_file_header_start = file_info['header_offset'] + len(local_file_header_signature_bytes)
         local_file_header_end = local_file_header_start + local_file_header_struct.size
 
-        local_file_header_response = requests.get(
+        local_file_header_response = HttpClient.request(
             url=lfs_url,
-            stream=True,
             headers={'range': f'bytes={local_file_header_start}-{local_file_header_end - 1}'},
-            timeout=300,
+            timeout_in_seconds=300,
         )
-        local_file_header_response.raise_for_status()
-        local_file_header_bytes: bytes = local_file_header_response.raw.data
-        local_file_header = LocalFileHeader._make(local_file_header_struct.unpack(local_file_header_bytes))
+        local_file_header = LocalFileHeader._make(local_file_header_struct.unpack(local_file_header_response.content))
 
         file_start = local_file_header_end + local_file_header.file_name_len + local_file_header.extra_field_len
         file_end = file_start + file_info['file_size']
 
-        response = requests.get(
+        response = HttpClient.request(
             url=lfs_url,
-            stream=True,
             headers={'range': f'bytes={file_start}-{file_end - 1}'},
-            timeout=300,  # timeout after 5 min
+            timeout_in_seconds=300,  # timeout after 5 min
         )
-        response.raise_for_status()
-        data: bytes = response.raw.data
-        return data
+        return response.content

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/utils/__init__.py

@@ -1,22 +1,19 @@
 import collections.abc
 import multiprocessing
 import os
-import time
 import socket
 import sys
 
 from typing import Optional
-import requests
 from importlib_metadata import version, PackageNotFoundError
 
 from biolib.utils.seq_util import SeqUtil, SeqUtilRecord
-
-# try fetching version, if it fails (usually when in dev), add default
-from biolib.biolib_errors import BioLibError
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_logging import logger_no_user_data, logger
 from biolib.typing_utils import Tuple, Iterator
 from .multipart_uploader import MultiPartUploader, get_chunk_iterator_from_bytes
 
+# try fetching version, if it fails (usually when in dev), add default
 try:
     BIOLIB_PACKAGE_VERSION = version('pybiolib')
 except PackageNotFoundError:
@@ -87,38 +84,17 @@ DownloadChunkInputTuple = Tuple[ByteRangeTuple, str]
 
 
 def _download_chunk(input_tuple: DownloadChunkInputTuple) -> bytes:
-    max_download_retries = 10
-
     byte_range, presigned_url = input_tuple
     start, end = byte_range
 
-    for retry_attempt in range(max_download_retries):
-        if retry_attempt > 0:
-            logger_no_user_data.debug(f'Attempt number {retry_attempt} for part {start}')
-        try:
-            response = requests.get(
-                url=presigned_url,
-                stream=True,
-                headers={'range': f'bytes={start}-{end}'},
-                timeout=300,  # timeout after 5 min
-            )
-            if response.ok:
-                return_value: bytes = response.raw.data
-                logger_no_user_data.debug(f'Returning raw data for part {start}')
-                return return_value
-            else:
-                logger_no_user_data.warning(
-                    f'Got not ok response when downloading part {start}:{end}. '
-                    f'Got response status {response.status_code} and content: {response.content.decode()} '
-                    f'Retrying...'
-                )
-        except Exception:  # pylint: disable=broad-except
-            logger_no_user_data.warning(f'Encountered error when downloading part {start}:{end}. Retrying...')
-
-        time.sleep(5)
-
-    logger_no_user_data.debug(f'Max retries hit, when downloading part {start}:{end}. Exiting...')
-    raise BioLibError(f'Max retries hit, when downloading part {start}:{end}. Exiting...')
+    response = HttpClient.request(
+        url=presigned_url,
+        headers={'range': f'bytes={start}-{end}'},
+        timeout_in_seconds=300,  # timeout after 5 min
+        retries=10,
+    )
+    logger_no_user_data.debug(f'Returning raw data for part {start}')
+    return response.content
 
 
 class ChunkIterator(collections.abc.Iterator):
@@ -153,11 +129,8 @@ class ChunkIterator(collections.abc.Iterator):
 def download_presigned_s3_url(presigned_url: str, output_file_path: str) -> None:
     chunk_size = 50_000_000
 
-    with requests.get(presigned_url, stream=True, headers={'range': 'bytes=0-1'}) as response:
-        if not response.ok:
-            raise Exception(f'Got response status code {response.status_code} and content {response.content.decode()}')
-
-        file_size = int(response.headers['Content-Range'].split('/')[1])
+    response = HttpClient.request(url=presigned_url, headers={'range': 'bytes=0-1'})
+    file_size = int(response.headers['Content-Range'].split('/')[1])
 
     chunk_iterator = ChunkIterator(file_size, chunk_size, presigned_url)
 

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/biolib/utils/multipart_uploader.py

@@ -5,10 +5,9 @@ import os
 import time
 from urllib.parse import urlparse
 
-import requests
-
+import biolib.api
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.auth import BearerAuth
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import TypedDict, List, Iterator, Tuple, Optional, Dict
@@ -33,7 +32,7 @@ def get_chunk_iterator_from_file_object(file_object, chunk_size_in_bytes: int =
 class RequestOptions(TypedDict):
     headers: Optional[Dict[str, str]]
     requires_biolib_auth: bool
-    url: str
+    path: str
 
 
 class _PartMetadata(TypedDict):
@@ -67,20 +66,15 @@ class MultiPartUploader:
         logger_no_user_data.debug(f'Starting multipart upload of payload with size {payload_size_in_bytes} bytes')
 
         if self._start_multipart_upload_request:
-            requires_biolib_auth = self._start_multipart_upload_request['requires_biolib_auth']
-            start_multipart_upload = requests.post(
-                auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
-                headers=self._start_multipart_upload_request['headers'],
-                timeout=30,
-                url=self._start_multipart_upload_request['url'],
-            )
-            if start_multipart_upload.ok:
-                logger_no_user_data.debug('Multipart upload started')
-            else:
-                logger_no_user_data.debug(
-                    f'Failed to start multipart upload got response status: {start_multipart_upload.status_code}'
+            try:
+                biolib.api.client.post(
+                    authenticate=self._start_multipart_upload_request['requires_biolib_auth'],
+                    headers=self._start_multipart_upload_request['headers'],
+                    path=self._start_multipart_upload_request['path'],
                 )
-                raise Exception('Failed to start multipart upload')
+            except BaseException as error:
+                logger_no_user_data.debug(f'Failed to start multipart upload got error: {error}')
+                raise error
 
         # if multiprocessing start method is spawn or we are running in a daemon process,
         # multiprocessing.Pool may fail when called from script
@@ -116,30 +110,12 @@ class MultiPartUploader:
             BiolibApiClient.refresh_auth_token()
 
         logger_no_user_data.debug(f'Uploaded {len(parts)} parts, now calling complete upload...')
-        for index in range(3):
-            try:
-                complete_upload_response = requests.post(
-                    auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
-                    headers=self._complete_upload_request['headers'],
-                    json={'parts': parts, 'size_bytes': self._bytes_uploaded},
-                    timeout=30,
-                    url=self._complete_upload_request['url'],
-                )
-                if complete_upload_response.ok:
-                    logger_no_user_data.debug('Multipart upload completed returning')
-                    return
-
-                logger_no_user_data.warning(
-                    f'Failed to complete multipart upload got response status {complete_upload_response.status_code}. '
-                    f'Retrying...'
-                )
-
-            except Exception as error:  # pylint: disable=broad-except
-                logger_no_user_data.warning('Encountered error when completing multipart upload. Retrying...')
-                logger.debug(f'Multipart complete error: {error}')
-                time.sleep(index * index + 2)
-
-        raise BioLibError('Max retries hit, when completing multipart upload')
+        biolib.api.client.post(
+            authenticate=requires_biolib_auth,
+            headers=self._complete_upload_request['headers'],
+            data={'parts': parts, 'size_bytes': self._bytes_uploaded},
+            path=self._complete_upload_request['path'],
+        )
 
     def _upload_chunk(self, _input: _UploadChunkInputType) -> _UploadChunkReturnType:
         part_number, chunk = _input
@@ -152,18 +128,12 @@ class MultiPartUploader:
             logger_no_user_data.debug(f'Uploading part number {part_number} with size {len(chunk)} bytes...')
             try:
                 logger_no_user_data.debug(f'Getting upload URL for chunk {part_number}...')
-                get_url_response = requests.get(
-                    auth=BearerAuth(BiolibApiClient.get().access_token) if requires_biolib_auth else None,
+                get_url_response = biolib.api.client.get(
+                    authenticate=requires_biolib_auth,
                     headers=self._get_presigned_upload_url_request['headers'],
                     params={'part_number': part_number},
-                    timeout=30,
-                    url=self._get_presigned_upload_url_request['url'],
+                    path=self._get_presigned_upload_url_request['path'],
                 )
-                if not get_url_response.ok:
-                    raise Exception(
-                        f'Failed to get upload URL for part {part_number} got response status code '
-                        f'{get_url_response.status_code}'
-                    )
 
                 presigned_upload_url = get_url_response.json()['presigned_upload_url']
 
@@ -174,16 +144,13 @@ class MultiPartUploader:
                     presigned_upload_url = \
                         f'{app_caller_proxy_job_storage_base_url}{parsed_url.path}?{parsed_url.query}'
 
-                put_chunk_response = requests.put(url=presigned_upload_url, data=chunk, timeout=300)
-
-                if put_chunk_response.ok:
-                    return _PartMetadata(PartNumber=part_number, ETag=put_chunk_response.headers['ETag']), len(chunk)
-                else:
-                    logger_no_user_data.warning(
-                        f'Got response with status {put_chunk_response.status_code} when uploading part {part_number}. '
-                        'Retrying...'
-                    )
-                    logger.debug(f'Response content: {put_chunk_response.content.decode()}')
+                put_chunk_response = HttpClient.request(
+                    url=presigned_upload_url,
+                    data=chunk,
+                    method='PUT',
+                    timeout_in_seconds=300,
+                )
+                return _PartMetadata(PartNumber=part_number, ETag=put_chunk_response.headers['ETag']), len(chunk)
 
             except Exception as error:  # pylint: disable=broad-except
                 logger_no_user_data.warning(f'Encountered error when uploading part {part_number}. Retrying...')

{pybiolib-1.1.1711 → pybiolib-1.1.1730}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.1.1711"
+version = "1.1.1730"
 description = "BioLib Python Client"
 readme = "README.md"
 license = "MIT"