pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/biolib_api_client/auth.py

@@ -1,11 +1,34 @@
-from requests.auth import AuthBase  # type: ignore
+from biolib import api
+from biolib.biolib_api_client.api_client import UserTokens
+from biolib.typing_utils import TypedDict, Literal
 
 
-class BearerAuth(AuthBase):
-    def __init__(self, access_token=None):
-        self.access_token = access_token
+class AuthChallengeCreate(TypedDict):
+    token: str
 
-    def __call__(self, req):
-        if self.access_token:
-            req.headers['Authorization'] = 'Bearer ' + self.access_token
-        return req
+
+class _AuthChallengeStatus(TypedDict):
+    state: Literal['awaiting', 'completed']
+
+
+class AuthChallengeStatus(_AuthChallengeStatus, total=False):
+    user_tokens: UserTokens
+
+
+class BiolibAuthChallengeApi:
+
+    @staticmethod
+    def create_auth_challenge() -> AuthChallengeCreate:
+        response = api.client.post(path='/user/auth_challenges/')
+        response_dict: AuthChallengeCreate = response.json()
+        return response_dict
+
+    @staticmethod
+    def get_auth_challenge_status(token: str) -> AuthChallengeStatus:
+        response = api.client.get(
+            path='/user/auth_challenges/',
+            headers={'Auth-Challenge-Token': token},
+        )
+
+        response_dict: AuthChallengeStatus = response.json()
+        return response_dict

biolib/biolib_api_client/biolib_app_api.py

@@ -1,71 +1,166 @@
+import mimetypes
+import os
+import random
 import re
+import subprocess
+import urllib.parse
 
-import requests
-
+import biolib.api
 from biolib import biolib_errors
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient, AppGetResponse
-from biolib.biolib_errors import BioLibError
+from biolib._internal.http_client import HttpError
+from biolib.api.client import ApiClient
+from biolib.biolib_api_client import AppGetResponse
 from biolib.biolib_logging import logger
+from biolib.typing_utils import Optional
+from biolib.utils import load_base_url_from_env
 
 
-class BiolibAppApi:
+def encode_multipart(data, files):
+    boundary = f'----------{random.randint(0, 1000000000)}'
+    line_array = []
 
-    @staticmethod
-    def get_by_uri(uri: str) -> AppGetResponse:
-        # Replace protocol with @ (if supplied)
-        uri = re.sub(r'^http(s)?://', '@', uri)
-        # Replace frontend version path with app_uri compatible version (if supplied)
-        uri = re.sub(r'/version/(?P<version>(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*))(/)?$', r':\g<version>', uri)
+    for key, value in data.items():
+        if value is not None:
+            line_array.append(f'--{boundary}')
+            line_array.append(f'Content-Disposition: form-data; name="{key}"')
+            line_array.append('')
+            line_array.append(value)
+
+    for key, (filename, value) in files.items():
+        line_array.append(f'--{boundary}')
+        line_array.append(f'Content-Disposition: form-data; name="{key}"; filename="{filename}"')
+        line_array.append(f'Content-Type: {mimetypes.guess_type(filename)[0] or "application/octet-stream"}')
+        line_array.append('')
+        line_array.append('')
+        line_array.append(value)
+
+    line_array.append(f'--{boundary}--')
+    line_array.append('')
+
+    data_encoded = b'\r\n'.join([line.encode() if isinstance(line, str) else line for line in line_array])
+    return f'multipart/form-data; boundary={boundary}', data_encoded
+
+
+def _get_git_branch_name() -> str:
+    try:
+        github_actions_branch_name = os.getenv('GITHUB_REF_NAME')
+        if github_actions_branch_name:
+            return github_actions_branch_name
+
+        gitlab_ci_branch_name = os.getenv('CI_COMMIT_REF_NAME')
+        if gitlab_ci_branch_name:
+            return gitlab_ci_branch_name
+
+        result = subprocess.run(['git', 'branch', '--show-current'], check=True, stdout=subprocess.PIPE, text=True)
+        return result.stdout.strip()
+    except BaseException:
+        return ''
+
+
+def _get_git_commit_hash() -> str:
+    try:
+        github_actions_commit_hash = os.getenv('GITHUB_SHA')
+        if github_actions_commit_hash:
+            return github_actions_commit_hash
 
-        response = requests.get(
-            f'{BiolibApiClient.get().base_url}/api/app/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            params={'uri': uri},
-            timeout=5,
-        )
+        gitlab_ci_commit_hash = os.getenv('CI_COMMIT_SHA')
+        if gitlab_ci_commit_hash:
+            return gitlab_ci_commit_hash
 
-        if response.status_code == 404:
-            raise biolib_errors.NotFound(f'Application {uri} not found.')
+        result = subprocess.run(['git', 'rev-parse', 'HEAD'], check=True, stdout=subprocess.PIPE, text=True)
+        return result.stdout.strip()
+    except BaseException:
+        return ''
 
-        if response.status_code == 400:
-            raise biolib_errors.BioLibError(response.content.decode())
 
-        if not response.ok:
-            raise Exception(response.content.decode())
+def _get_git_repository_url() -> str:
+    try:
+        result = subprocess.run(['git', 'remote', 'get-url', 'origin'], check=True, stdout=subprocess.PIPE, text=True)
+        return result.stdout.strip()
+    except BaseException:
+        return ''
 
-        app_response: AppGetResponse = response.json()
-        return app_response
 
+def _get_resource_uri_from_str(input_str: str) -> str:
+    parsed_base_url = urllib.parse.urlparse(load_base_url_from_env())
+    parsed_uri = urllib.parse.urlparse(input_str)
+    if parsed_uri.netloc != '' and parsed_base_url.netloc != parsed_uri.netloc:
+        raise biolib_errors.ValidationError(f'Invalid URI. The hostname "{parsed_base_url.netloc}" is not recognized.')
+    elif parsed_uri.netloc != '' and parsed_uri.path[1] != '@':
+        uri = f'@{parsed_uri.netloc}{parsed_uri.path}'
+    elif parsed_uri.netloc == '' and parsed_uri.path.startswith(parsed_base_url.netloc):
+        uri = f'@{parsed_uri.path}'
+    else:
+        uri = parsed_uri.path
+    uri = uri.strip('/')
+    # Replace frontend version path with app_uri compatible version (if supplied)
+    uri = re.sub(r'/version/(?P<version>(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*))(/)?$', r':\g<version>', uri)
+    return uri
+
+
+class BiolibAppApi:
     @staticmethod
-    def push_app_version(app_id, zip_binary, author, app_name, set_as_active):
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/app_versions/',
-            files={
-                'app': (None, app_id),
-                'set_as_active': (None, 'true' if set_as_active else 'false'),
-                'state': (None, 'published'),
-                'source_files_zip': zip_binary
-            },
-            auth=BearerAuth(BiolibApiClient.get().access_token)
-        )
-        if not response.ok:
-            logger.error(f'Push failed for {author}/{app_name}:')
-            raise BioLibError(response.text)
+    def get_by_uri(uri: str, api_client: Optional[ApiClient] = None) -> AppGetResponse:
+        uri = _get_resource_uri_from_str(uri)
+        api = api_client or biolib.api.client
+        try:
+            response = api.get(path='/app/', params={'uri': uri})
+            app_response: AppGetResponse = response.json()
+            return app_response
 
-        # TODO: When response includes the version number, print the URL for the new app version
-        logger.info(f'Successfully pushed app version for {author}/{app_name}.')
-        return response.json()
+        except HttpError as error:
+            if error.code == 404:
+                raise biolib_errors.NotFound(f'Application {uri} not found.') from None
+
+            raise error
+
+    @staticmethod
+    def create_app(uri: str):
+        uri = _get_resource_uri_from_str(uri)
+        try:
+            response = biolib.api.client.post(path='/resources/apps/', data={'uri': uri})
+            return response.json()
+        except HttpError as error:
+            raise error
 
     @staticmethod
-    def update_app_version(app_version_id, data):
-        response = requests.patch(
-            f'{BiolibApiClient.get().base_url}/api/app_versions/{app_version_id}/',
-            json=data,
-            auth=BearerAuth(BiolibApiClient.get().access_token)
-        )
-        if not response.ok:
-            logger.error(f'Failed to update app version {app_version_id}')
-            raise BioLibError(response.text)
+    def push_app_version(
+        app_id,
+        zip_binary,
+        author,
+        app_name,
+        set_as_active,
+        app_version_id_to_copy_images_from: Optional[str],
+        semantic_version: Optional[str],
+    ):
+        try:
+            data = {
+                'app': app_id,
+                'set_as_active': 'true' if set_as_active else 'false',
+                'state': 'published',
+                'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
+                'git_branch_name': _get_git_branch_name(),
+                'git_commit_hash': _get_git_commit_hash(),
+                'git_repository_url': _get_git_repository_url(),
+            }
+            if semantic_version:
+                data['semantic_version'] = semantic_version
 
+            content_type, data_encoded = encode_multipart(
+                data=data,
+                files={
+                    'source_files_zip': ('source_files.zip', zip_binary),
+                },
+            )
+            response = biolib.api.client.post(
+                path='/app_versions/',
+                data=data_encoded,
+                headers={'Content-Type': content_type},
+            )
+        except Exception as error:
+            logger.error(f'Push failed for {author}/{app_name}:')
+            raise error
+
+        # TODO: When response includes the version number, print the URL for the new app version
+        logger.info(f'Initialized new app version for {author}/{app_name}.')
         return response.json()

biolib/biolib_api_client/biolib_job_api.py

@@ -1,172 +1,192 @@
-import time
-import requests
+import os
+from urllib.parse import urlparse
 
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_errors import BioLibError
-from biolib.compute_node.utils import SystemExceptionCodeMap
+import biolib.api
+
+from biolib import utils
+from biolib._internal.http_client import HttpError
+from biolib.api.client import ApiClient
+from biolib.biolib_api_client import CloudJob, JobState
+from biolib.biolib_errors import JobResultPermissionError, JobResultError, JobResultNotFound, StorageDownloadFailed
 from biolib.biolib_logging import logger
 from biolib.utils import BIOLIB_PACKAGE_VERSION
+from biolib.typing_utils import TypedDict, Optional, Literal, Dict
 
 
-class RetryException(Exception):
-    pass
-
-
-class BiolibJobApi:
-
-    @staticmethod
-    def create(app_version_id, caller_job=None):
-        data = {
-                'app_version_id': app_version_id,
-                'client_type': 'biolib-python',
-                'client_version': BIOLIB_PACKAGE_VERSION,
-        }
-        if caller_job:
-            data['caller_job'] = caller_job
-
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/jobs/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            json=data,
-        )
+class PresignedS3UploadLinkResponse(TypedDict):
+    presigned_upload_url: str
 
-        # TODO: Error handling with response object
-        if not response.ok:
-            raise BioLibError(response.content)
 
-        return response.json()
+class PresignedS3DownloadLinkResponse(TypedDict):
+    presigned_download_url: str
 
-    @staticmethod
-    def update_state(job_id, state):
-        response = requests.patch(
-            f'{BiolibApiClient.get().base_url}/api/jobs/{job_id}/',
-            json={'state': state},
-            auth=BearerAuth(BiolibApiClient.get().access_token)
-        )
 
-        # TODO: Error handling with response object
-        if not response.ok:
-            raise BioLibError(response.content)
+def _get_user_info() -> Optional[str]:
+    if utils.BASE_URL_IS_PUBLIC_BIOLIB:
+        return None
 
-        return response.json()
+    enterprise_agent_info_opt_env_vars = ['BIOLIB_OPT_USER', 'DOMINO_STARTING_USERNAME', 'USER']
 
-    @staticmethod
-    def create_cloud_job(module_name, job_id):
-        response = None
-        for retry in range(4):
-            try:
-                response = requests.post(
-                    f'{BiolibApiClient.get().base_url}/api/jobs/cloud/',
-                    json={'module_name': module_name, 'job_id': job_id},
-                    auth=BearerAuth(BiolibApiClient.get().access_token)
-                )
-
-                if response.status_code == 503:
-                    raise RetryException(response.content)
-                # Handle possible validation errors from backend
-                elif not response.ok:
-                    raise BioLibError(response.text)
-
-                break
-
-            except RetryException as retry_exception:  # pylint: disable=broad-except
-                if retry > 3:
-                    raise BioLibError('Reached retry limit for cloud job creation') from retry_exception
-                time.sleep(1)
-
-        if not response:
-            raise BioLibError('Could not create new cloud job')
-
-        cloud_job = response.json()
-        if cloud_job.get('is_compute_node_ready', False):
-            return cloud_job
-
-        max_retry_attempts = 25
-        retry_interval_seconds = 10
-
-        for _ in range(max_retry_attempts):
-            response = requests.get(
-                f'{BiolibApiClient.get().base_url}/api/jobs/cloud/{cloud_job["public_id"]}/status/',
-                auth=BearerAuth(BiolibApiClient.get().access_token)
-            )
-            cloud_job = response.json()
-            if cloud_job.get('is_compute_node_ready', False):
-                return cloud_job
+    for env_var in enterprise_agent_info_opt_env_vars:
+        env_var_value = os.getenv(env_var)
+        if env_var_value:
+            return env_var_value
 
-            logger.info('Cloud: Reserved compute node not ready, retrying...')
-            time.sleep(retry_interval_seconds)
+    return None
 
-        raise BioLibError('Cloud: The reserved compute node was not ready in time')
-
-    @staticmethod
-    def save_compute_node_job(job, module_name, access_token, node_url):
-        response = requests.post(
-            f'{node_url}/v1/job/',
-            json={'module_name': module_name, 'job': job, 'access_token': access_token}
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content)
-
-        return response.json
 
+class BiolibJobApi:
     @staticmethod
-    def start_cloud_job(job_id, module_input_serialized, node_url):
-        response = requests.post(
-            f'{node_url}/v1/job/{job_id}/start/',
-            data=module_input_serialized
-        )
+    def create(
+        app_version_id,
+        app_resource_name_prefix=None,
+        override_command=False,
+        caller_job=None,
+        machine='',
+        experiment_uuid: Optional[str] = None,
+        timeout: Optional[int] = None,
+        notify: bool = False,
+        requested_machine_count: Optional[int] = None,
+        temporary_client_secrets: Optional[Dict[str, str]] = None,
+        api_client: Optional[ApiClient] = None,
+    ):
+        data = {
+            'app_version_id': app_version_id,
+            'client_type': 'biolib-python',
+            'notify': notify,
+            'client_version': BIOLIB_PACKAGE_VERSION,
+            'client_opt_user_info': _get_user_info(),
+        }
 
-        if not response.ok:
-            raise BioLibError(response.content)
+        if app_resource_name_prefix:
+            data.update({'app_resource_name_prefix': app_resource_name_prefix})
 
-    @staticmethod
-    def await_compute_node_status(retry_interval_seconds, retry_limit_minutes, status_to_await,
-                                  compute_type, job_id, node_url):
-        status_max_retry_attempts = int(retry_limit_minutes * 60 / retry_interval_seconds)
-        status_reached = False
-        for _ in range(status_max_retry_attempts):
-            response = requests.get(f'{node_url}/v1/job/{job_id}/status/')
-            if not response.ok:
-                raise Exception(response.content)
+        if override_command:
+            data.update({'arguments_override_command': override_command})
 
-            status_json = response.json()
+        if caller_job:
+            data['caller_job'] = caller_job
 
-            for status_update in status_json['status_updates']:
-                if 'log_message' in status_update:
-                    logger.info(f'{compute_type}: {status_update["log_message"]}')
+        if machine:
+            data.update({'requested_machine': machine})
 
-                if status_update.get('log_message') == status_to_await:
-                    status_reached = True
+        if requested_machine_count:
+            data.update({'requested_machine_count': requested_machine_count})
 
-            if 'error_code' in status_json:
-                error_code = status_json['error_code']
-                error_message = SystemExceptionCodeMap.get(error_code, f'Unknown error code {error_code}')
+        if experiment_uuid:
+            data['experiment_uuid'] = experiment_uuid
 
-                raise BioLibError(f'{compute_type}: {error_message}')
+        if timeout:
+            data['requested_timeout_seconds'] = timeout
 
-            if status_reached:
-                return
+        if temporary_client_secrets:
+            data['temporary_client_secrets'] = temporary_client_secrets
 
-            time.sleep(retry_interval_seconds)
+        api = api_client or biolib.api.client
+        response = api.post(path='/jobs/', data=data)
 
-        raise BioLibError(f'{compute_type}: Failed to get results: Retry limit exceeded')
+        return response.json()
 
     @staticmethod
-    def get_cloud_result(job_id, node_url):
-        response = requests.get(
-            f'{node_url}/v1/job/{job_id}/result/',
-        )
+    def update_state(job_uuid: str, state: JobState) -> None:
+        try:
+            biolib.api.client.patch(path=f'/jobs/{job_uuid}/', data={'state': state.value})
+        except BaseException as error:
+            logger.error(f'Failed to update job "{job_uuid}" to state "{state.value}" due to {error}')
 
-        if not response.ok:
-            raise Exception(response.content)
+    @staticmethod
+    def create_cloud_job(
+        job_id: str,
+        result_name_prefix: Optional[str],
+        api_client: Optional[ApiClient] = None,
+    ) -> CloudJob:
+        data = {'job_id': job_id}
+        if result_name_prefix:
+            data['result_name_prefix'] = result_name_prefix
+
+        api = api_client or biolib.api.client
+        response = api.post(path='/jobs/cloud/', data=data)
+        cloud_job: CloudJob = response.json()
+        return cloud_job
 
-        return response.content
+    @staticmethod
+    def get_job_storage_download_url(
+        job_uuid: str,
+        job_auth_token: str,
+        storage_type: Literal['input', 'results'],
+    ) -> str:
+        try:
+            response = biolib.api.client.get(
+                path=f'/jobs/{job_uuid}/storage/{storage_type}/download/',
+                authenticate=True,
+                headers={'Job-Auth-Token': job_auth_token},
+            )
+            presigned_s3_download_link_response: PresignedS3DownloadLinkResponse = response.json()
+            presigned_download_url = presigned_s3_download_link_response['presigned_download_url']
+
+            app_caller_proxy_job_storage_base_url = os.getenv('BIOLIB_CLOUD_JOB_STORAGE_BASE_URL', '')
+            if app_caller_proxy_job_storage_base_url:
+                # Done to hit App Caller Proxy when downloading result from inside an app
+                parsed_url = urlparse(presigned_download_url)
+                presigned_download_url = f'{app_caller_proxy_job_storage_base_url}{parsed_url.path}?{parsed_url.query}'
+
+            return presigned_download_url
+
+        except HttpError as error:
+            if storage_type == 'results':
+                if error.code == 401:
+                    raise JobResultPermissionError('You must be signed in to get result of the job') from None
+                elif error.code == 403:
+                    raise JobResultPermissionError(
+                        'Cannot get result of job. Maybe the job was created without being signed in?'
+                    ) from None
+                elif error.code == 404:
+                    raise JobResultNotFound('Job result not found') from None
+                else:
+                    raise JobResultError('Failed to get result of job') from error
+            else:
+                raise StorageDownloadFailed(f'Failed to download result of job got error: {error}') from error
+
+        except Exception as error:  # pylint: disable=broad-except
+            if storage_type == 'results':
+                raise JobResultError('Failed to get result of job') from error
+            else:
+                raise StorageDownloadFailed('Failed to download from Job Storage') from error
 
     @staticmethod
-    def get_enclave_json(biolib_base_url):
-        response = requests.get(
-            f'{biolib_base_url}/info-files/biolib-enclave.json',
-        )
-        return response.json()
+    def create_job_with_data(
+        app_version_uuid: str,
+        app_resource_name_prefix: Optional[str],
+        module_input_serialized: bytes,
+        arguments_override_command: bool,
+        experiment_uuid: Optional[str],
+        requested_machine: Optional[str],
+        result_name_prefix: Optional[str],
+        caller_job_uuid: Optional[str] = None,
+        requested_timeout_seconds: Optional[int] = None,
+        notify: bool = False,
+        requested_machine_count: Optional[int] = None,
+        api_client: Optional[ApiClient] = None,
+    ) -> Dict:
+        api = api_client or biolib.api.client
+        job_dict: Dict = api.post(
+            path='/jobs/create_job_with_data/',
+            data=module_input_serialized,
+            headers={
+                'Content-Type': 'application/octet-stream',
+                'app-version-uuid': app_version_uuid,
+                'app-resource-name-prefix': app_resource_name_prefix,
+                'arguments-override-command': str(arguments_override_command),
+                'caller-job-uuid': caller_job_uuid,
+                'client-opt-user-info': _get_user_info(),
+                'client-type': 'biolib-python',
+                'client-version': BIOLIB_PACKAGE_VERSION,
+                'experiment-uuid': experiment_uuid,
+                'requested-machine': requested_machine,
+                'requested-machine-count': str(requested_machine_count) if requested_machine_count else None,
+                'result-name-prefix': result_name_prefix,
+                'requested-timeout-seconds': str(requested_timeout_seconds) if requested_timeout_seconds else None,
+                'notify': 'true' if notify else 'false',
+            },
+        ).json()
+        return job_dict

biolib/biolib_api_client/job_types.py

@@ -1,9 +1,8 @@
 from enum import Enum
 
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Optional, List
-
 from biolib.biolib_api_client.app_types import AppVersionOnJob, RemoteHost
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Dict, List, Optional, TypedDict
 
 
 class JobState(Enum):
@@ -15,21 +14,38 @@ class JobState(Enum):
 
 
 class _Job(TypedDict):
+    app_uri: str
     app_version: AppVersionOnJob
+    arguments_override_command: bool
+    auth_token: str
     caller_job: Optional[str]
     created_at: str
+    federated_job_uuid: Optional[str]
     public_id: str
     remote_hosts_with_warning: List[RemoteHost]
+    state: str
     user_id: Optional[str]
+    uuid: str
 
 
 # type optional keys with total=False
-class Job(_Job, total=False):
+class CreatedJobDict(_Job, total=False):
     custom_compute_node_url: str
+    temporary_client_secrets: Dict[str, str]
+
+
+class CloudJob(TypedDict):
+    public_id: str
+    reserved_cpu_in_nano_shares: int
+    reserved_gpu_count: int
+    reserved_memory_in_bytes: int
+    max_runtime_in_seconds: int
 
 
 class JobWrapper(TypedDict):
     access_token: str
     BASE_URL: str  # TODO: refactor this to lower case
     compute_node_info: Optional[ComputeNodeInfo]
-    job: Job
+    job: CreatedJobDict
+    cloud_job: Optional[CloudJob]
+    job_temporary_dir: Optional[str]