pybiolib 1.1.1747__py3-none-any.whl → 1.1.2193__py3-none-any.whl

biolib/_internal/push_application.py

@@ -1,17 +1,18 @@
 import os
 import re
 from pathlib import Path
-import yaml
+
 import rich.progress
+import yaml
 
-from biolib.lfs.utils import get_iterable_zip_stream, get_files_and_size_of_directory
-from biolib.typing_utils import Optional, Set, TypedDict, Iterable
+from biolib import api, utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
-from biolib import utils, api
+from biolib.typing_utils import Iterable, Optional, Set, TypedDict
 
 REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')
 
@@ -38,9 +39,7 @@ def process_docker_status_updates(status_updates: Iterable[DockerStatusUpdate],
                 progress_detail = update['progressDetail']
 
                 if layer_id not in layer_id_to_task_id:
-                    layer_id_to_task_id[layer_id] = progress.add_task(
-                        description=f'[cyan]{action} layer {layer_id}'
-                    )
+                    layer_id_to_task_id[layer_id] = progress.add_task(description=f'[cyan]{action} layer {layer_id}')
 
                 if progress_detail and 'current' in progress_detail and 'total' in progress_detail:
                     progress.update(
@@ -60,7 +59,7 @@ def process_docker_status_updates(status_updates: Iterable[DockerStatusUpdate],
 
 
 def set_app_version_as_active(
-        app_version_uuid: str,
+    app_version_uuid: str,
 ):
     logger.debug(f'Setting app version {app_version_uuid} as active.')
     api.client.patch(
@@ -70,10 +69,10 @@ def set_app_version_as_active(
 
 
 def push_application(
-        app_uri: str,
-        app_path: str,
-        app_version_to_copy_images_from: Optional[str],
-        is_dev_version: Optional[bool],
+    app_uri: str,
+    app_path: str,
+    app_version_to_copy_images_from: Optional[str],
+    is_dev_version: Optional[bool],
 ):
     app_path_absolute = Path(app_path).resolve()
 
@@ -96,7 +95,7 @@ def push_application(
 
     input_files_maps_to_root = False
     try:
-        with open(config_yml_path, mode='r') as config_yml_file:
+        with open(config_yml_path) as config_yml_file:
             config = yaml.safe_load(config_yml_file.read())
 
         license_file_relative_path = config.get('license_file', 'LICENSE')
@@ -109,7 +108,7 @@ def push_application(
             raise BioLibError(f'Could not find {description_file_relative_path}')
 
         zip_filters.add(description_file_relative_path)
-        with open(description_file_absolute_path, mode='r') as description_file:
+        with open(description_file_absolute_path) as description_file:
             description_file_content = description_file.read()
 
         for _, img_src_path in re.findall(REGEX_MARKDOWN_INLINE_IMAGE, description_file_content):
@@ -171,8 +170,9 @@ def push_application(
         author=app['account_handle'],
         set_as_active=False,
         zip_binary=source_files_zip_bytes,
-        app_version_id_to_copy_images_from=app_response['app_version']['public_id'] if app_version_to_copy_images_from
-        else None
+        app_version_id_to_copy_images_from=app_response['app_version']['public_id']
+        if app_version_to_copy_images_from
+        else None,
     )
 
     #  Don't push docker images if copying from another app version
@@ -180,18 +180,6 @@ def push_application(
     if not app_version_to_copy_images_from and docker_tags:
         logger.info('Found docker images to push.')
 
-        try:
-            yaml_file = open(f'{app_path}/.biolib/config.yml', 'r', encoding='utf-8')
-
-        except Exception as error:  # pylint: disable=broad-except
-            raise BioLibError('Could not open the config file .biolib/config.yml') from error
-
-        try:
-            config_data = yaml.safe_load(yaml_file)
-
-        except Exception as error:  # pylint: disable=broad-except
-            raise BioLibError('Could not parse .biolib/config.yml. Please make sure it is valid YAML') from error
-
         # Auth to be sent to proxy
         # The tokens are sent as "{access_token},{job_id}". We leave job_id blank on push.
         tokens = f'{BiolibApiClient.get().access_token},'
@@ -200,14 +188,12 @@ def push_application(
         docker_client = BiolibDockerClient.get_docker_client()
 
         for module_name, repo_and_tag in docker_tags.items():
-            docker_image_definition = config_data['modules'][module_name]['image']
+            docker_image_definition = config['modules'][module_name]['image']
             repo, tag = repo_and_tag.split(':')
 
             if docker_image_definition.startswith('dockerhub://'):
                 docker_image_name = docker_image_definition.replace('dockerhub://', 'docker.io/', 1)
-                logger.info(
-                    f'Pulling image {docker_image_name} defined on module {module_name} from Dockerhub.'
-                )
+                logger.info(f'Pulling image {docker_image_name} defined on module {module_name} from Dockerhub.')
                 dockerhub_repo, dockerhub_tag = docker_image_name.split(':')
                 pull_status_updates: Iterable[DockerStatusUpdate] = docker_client.api.pull(
                     decode=True,
@@ -238,7 +224,7 @@ def push_application(
 
                 process_docker_status_updates(push_status_updates, action='Pushing')
 
-            except Exception as exception:  # pylint: disable=broad-except
+            except Exception as exception:
                 raise BioLibError(f'Failed to tag and push image {docker_image_name}.') from exception
 
             logger.info(f'Successfully pushed {docker_image_name}')
@@ -249,10 +235,9 @@ def push_application(
         data={'set_as_active': not is_dev_version},
     )
 
-    sematic_version = \
-        f"{new_app_version_json['major']}.{new_app_version_json['minor']}.{new_app_version_json['patch']}"
+    sematic_version = f"{new_app_version_json['major']}.{new_app_version_json['minor']}.{new_app_version_json['patch']}"
     logger.info(
         f"Successfully pushed new {'development ' if is_dev_version else ''}version {sematic_version} of {app_uri}."
     )
 
-    return {"app_uri": app_uri, "sematic_version": sematic_version}
+    return {'app_uri': app_uri, 'sematic_version': sematic_version}

biolib/_internal/runtime.py

@@ -0,0 +1,19 @@
+from biolib.typing_utils import TypedDict
+
+
+class RuntimeJobDataDict(TypedDict):
+    version: str
+    job_requested_machine: str
+    job_uuid: str
+    job_auth_token: str
+    app_uri: str
+
+
+class BioLibRuntimeError(Exception):
+    pass
+
+
+class BioLibRuntimeNotRecognizedError(BioLibRuntimeError):
+    def __init__(self, message='The runtime is not recognized as a BioLib app'):
+        self.message = message
+        super().__init__(self.message)

biolib/_internal/types/__init__.py

@@ -0,0 +1,4 @@
+from .app import *  # noqa: F403
+from .data_record import *  # noqa: F403
+from .experiment import *  # noqa: F403
+from .resource import *  # noqa: F403

biolib/_internal/types/app.py

@@ -0,0 +1,9 @@
+from .typing import TypedDict
+
+
+class AppSlimDict(TypedDict):
+    pass
+
+
+class AppDetailedDict(AppSlimDict):
+    pass

biolib/_internal/types/data_record.py

@@ -0,0 +1,40 @@
+from .typing import Dict, List, Literal, Optional, TypedDict, Union
+
+
+class SqliteV1ForeignKey(TypedDict):
+    table: str
+    column: str
+
+
+class SqliteV1Column(TypedDict):
+    type: Literal['INTEGER', 'REAL', 'TEXT', 'JSON']  # noqa:F821
+    nullable: Optional[bool]
+    foreign_key: Optional[SqliteV1ForeignKey]
+    json_schema: Optional[Dict]
+
+
+class SqliteV1Table(TypedDict):
+    columns: Dict[str, SqliteV1Column]
+
+
+class SqliteV1DatabaseSchema(TypedDict):
+    tables: Dict[str, SqliteV1Table]
+
+
+class DataRecordValidationRuleDict(TypedDict):
+    path: str
+    type: str
+    rule: Union[SqliteV1DatabaseSchema]
+
+
+class DataRecordTypeDict(TypedDict):
+    name: str
+    validation_rules: List[DataRecordValidationRuleDict]
+
+
+class DataRecordSlimDict(TypedDict):
+    pass
+
+
+class DataRecordDetailedDict(DataRecordSlimDict):
+    type: Optional[DataRecordTypeDict]

biolib/_internal/types/experiment.py

@@ -0,0 +1,10 @@
+from .typing import TypedDict
+
+
+class ExperimentSlimDict(TypedDict):
+    job_count: int
+    job_running_count: int
+
+
+class ExperimentDetailedDict(ExperimentSlimDict):
+    pass

biolib/_internal/types/resource.py

@@ -0,0 +1,14 @@
+from .app import AppSlimDict
+from .data_record import DataRecordSlimDict
+from .experiment import ExperimentSlimDict
+from .typing import Optional, TypedDict
+
+
+class ResourceDict(TypedDict):
+    uuid: str
+    uri: str
+    name: str
+    created_at: str
+    app: Optional[AppSlimDict]
+    data_record: Optional[DataRecordSlimDict]
+    experiment: Optional[ExperimentSlimDict]

biolib/_internal/types/typing.py

@@ -0,0 +1,7 @@
+import sys
+
+# import and expose everything from the typing module
+from typing import *  # noqa:F403 pylint: disable=wildcard-import, unused-wildcard-import
+
+if sys.version_info < (3, 8):  # noqa: UP036
+    from typing_extensions import Literal, TypedDict  # pylint: disable=unused-import

biolib/_internal/utils/__init__.py

@@ -0,0 +1,18 @@
+import time
+import uuid
+
+
+def open_browser_window_from_notebook(url_to_open: str) -> None:
+    try:
+        from IPython.display import (  # type:ignore # pylint: disable=import-error, import-outside-toplevel
+            Javascript,
+            display,
+            update_display,
+        )
+    except ImportError as error:
+        raise Exception('Unexpected environment. This function can only be called from a notebook.') from error
+
+    display_id = str(uuid.uuid4())
+    display(Javascript(f'window.open("{url_to_open}");'), display_id=display_id)
+    time.sleep(1)
+    update_display(Javascript(''), display_id=display_id)

biolib/_runtime/runtime.py

@@ -0,0 +1,80 @@
+import json
+import re
+from typing import Optional
+
+from biolib import api
+from biolib._internal.runtime import BioLibRuntimeError, BioLibRuntimeNotRecognizedError, RuntimeJobDataDict
+from biolib.typing_utils import cast
+
+
+class Runtime:
+    _job_data: Optional[RuntimeJobDataDict] = None
+
+    @staticmethod
+    def check_is_environment_biolib_app() -> bool:
+        return bool(Runtime._try_to_get_job_data())
+
+    @staticmethod
+    def get_job_id() -> str:
+        return Runtime._get_job_data()['job_uuid']
+
+    @staticmethod
+    def get_job_auth_token() -> str:
+        return Runtime._get_job_data()['job_auth_token']
+
+    @staticmethod
+    def get_job_requested_machine() -> str:
+        return Runtime._get_job_data()['job_requested_machine']
+
+    @staticmethod
+    def get_app_uri() -> str:
+        return Runtime._get_job_data()['app_uri']
+
+    @staticmethod
+    def get_secret(secret_name: str) -> bytes:
+        assert re.match(
+            '^[a-zA-Z0-9_-]*$', secret_name
+        ), 'Secret name can only contain alphanumeric characters and dashes or underscores '
+        try:
+            with open(f'/biolib/secrets/{secret_name}', 'rb') as file:
+                return file.read()
+        except BaseException as error:
+            raise BioLibRuntimeError(f'Unable to get system secret: {secret_name}') from error
+
+    @staticmethod
+    def set_main_result_prefix(result_prefix: str) -> None:
+        job_data = Runtime._get_job_data()
+        api.client.patch(
+            data={'result_name_prefix': result_prefix},
+            headers={'Job-Auth-Token': job_data['job_auth_token']},
+            path=f"/jobs/{job_data['job_uuid']}/main_result/",
+        )
+
+    @staticmethod
+    def create_result_note(note: str) -> None:
+        job_id = Runtime.get_job_id()
+        # Note: Authentication is added by app caller proxy in compute node
+        api.client.post(data={'note': note}, path=f'/jobs/{job_id}/notes/')
+
+    @staticmethod
+    def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
+        if not Runtime._job_data:
+            try:
+                with open('/biolib/secrets/biolib_system_secret') as file:
+                    job_data: RuntimeJobDataDict = json.load(file)
+            except BaseException:
+                return None
+
+            if not job_data['version'].startswith('1.'):
+                raise BioLibRuntimeError(f"Unexpected system secret version {job_data['version']} expected 1.x.x")
+
+            Runtime._job_data = job_data
+
+        return cast(RuntimeJobDataDict, Runtime._job_data)
+
+    @staticmethod
+    def _get_job_data() -> RuntimeJobDataDict:
+        job_data = Runtime._try_to_get_job_data()
+        if not job_data:
+            raise BioLibRuntimeNotRecognizedError() from None
+        return job_data

biolib/api/__init__.py

@@ -1,4 +1,5 @@
 from biolib._internal.http_client import HttpClient
+
 from .client import ApiClient as _ApiClient
 
 _client = _ApiClient()

biolib/api/client.py

@@ -1,8 +1,10 @@
-from urllib.parse import urljoin, urlencode
+from urllib.parse import urlencode, urljoin
 
-from biolib.typing_utils import Dict, Optional, Union
+import importlib_metadata
+
+from biolib._internal.http_client import HttpClient, HttpResponse
 from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
-from biolib._internal.http_client import HttpResponse, HttpClient
+from biolib.typing_utils import Dict, Optional, Union, cast
 
 OptionalHeaders = Union[
     Optional[Dict[str, str]],
@@ -10,14 +12,24 @@ OptionalHeaders = Union[
 ]
 
 
+def _get_biolib_package_version() -> str:
+    # try fetching version, if it fails (usually when in dev), add default
+    try:
+        return cast(str, importlib_metadata.version('pybiolib'))
+    except importlib_metadata.PackageNotFoundError:
+        return '0.0.0'
+
+
 class ApiClient(HttpClient):
+    _biolib_package_version: str = _get_biolib_package_version()
+
     def get(
-            self,
-            path: str,
-            params: Optional[Dict[str, Union[str, int]]] = None,
-            headers: OptionalHeaders = None,
-            authenticate: bool = True,
-            retries: int = 10,
+        self,
+        path: str,
+        params: Optional[Dict[str, Union[str, int]]] = None,
+        headers: OptionalHeaders = None,
+        authenticate: bool = True,
+        retries: int = 10,
     ) -> HttpResponse:
         return self.request(
             headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
@@ -27,12 +39,12 @@ class ApiClient(HttpClient):
         )
 
     def post(
-            self,
-            path: str,
-            data: Optional[Union[Dict, bytes]] = None,
-            headers: OptionalHeaders = None,
-            authenticate: bool = True,
-            retries: int = 5,
+        self,
+        path: str,
+        data: Optional[Union[Dict, bytes]] = None,
+        headers: OptionalHeaders = None,
+        authenticate: bool = True,
+        retries: int = 50,  # TODO: reduce this back to 5 when timeout errors have been solved
     ) -> HttpResponse:
         return self.request(
             data=data,
@@ -42,13 +54,20 @@ class ApiClient(HttpClient):
             url=self._get_absolute_url(path=path, query_params=None),
         )
 
-    def patch(self, path: str, data: Dict, headers: OptionalHeaders = None, retries: int = 5) -> HttpResponse:
+    def patch(
+        self,
+        path: str,
+        data: Dict,
+        headers: OptionalHeaders = None,
+        retries: int = 5,
+        params: Optional[Dict[str, Union[str, int]]] = None,
+    ) -> HttpResponse:
         return self.request(
             data=data,
             headers=self._get_headers(opt_headers=headers, authenticate=True),
             method='PATCH',
             retries=retries,
-            url=self._get_absolute_url(path=path, query_params=None),
+            url=self._get_absolute_url(path=path, query_params=params),
         )
 
     @staticmethod
@@ -67,6 +86,9 @@ class ApiClient(HttpClient):
         if access_token and authenticate:
             headers['Authorization'] = f'Bearer {access_token}'
 
+        headers['client-type'] = 'biolib-python'
+        headers['client-version'] = ApiClient._biolib_package_version
+
         return headers
 
     @staticmethod

biolib/app/app.py

@@ -1,29 +1,26 @@
-import os
 import io
-import random
 import json
+import os
+import random
 import string
-
 from pathlib import Path
+
 from biolib import utils
-from biolib.compute_node.job_worker.job_storage import JobStorage
-from biolib.compute_node.job_worker.job_worker import JobWorker
-from biolib.experiments.experiment import Experiment
-from biolib.jobs import Job
-from biolib.typing_utils import Optional, cast
-from biolib.biolib_api_client import CreatedJobDict, JobState
-from biolib.jobs.types import JobDict
+from biolib.biolib_api_client import JobState
 from biolib.biolib_api_client.app_types import App, AppVersion
-from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import ModuleInput
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
+from biolib.compute_node.job_worker.job_worker import JobWorker
+from biolib.experiments.experiment import Experiment
+from biolib.jobs import Job
+from biolib.typing_utils import Optional
 from biolib.utils.app_uri import parse_app_uri
 
 
 class BioLibApp:
-
     def __init__(self, uri: str):
         app_response = BiolibAppApi.get_by_uri(uri)
         self._app: App = app_response['app']
@@ -48,17 +45,18 @@ class BioLibApp:
         return self._app_version
 
     def cli(
-            self,
-            args=None,
-            stdin=None,
-            files=None,
-            override_command=False,
-            machine='',
-            blocking: bool = True,
-            experiment_id: Optional[str] = None,
-            result_prefix: Optional[str] = None,
-            timeout: Optional[int] = None,
-            notify: bool = False,
+        self,
+        args=None,
+        stdin=None,
+        files=None,
+        override_command=False,
+        machine='',
+        blocking: bool = True,
+        experiment_id: Optional[str] = None,
+        result_prefix: Optional[str] = None,
+        timeout: Optional[int] = None,
+        notify: bool = False,
+        machine_count: Optional[int] = None,
     ) -> Job:
         if not experiment_id:
             experiment = Experiment.get_experiment_in_context()
@@ -78,7 +76,9 @@ class BioLibApp:
 
             return self._run_locally(module_input_serialized)
 
-        job = self._start_in_cloud(
+        job = Job._start_job_in_cloud(  # pylint: disable=protected-access
+            app_uri=self._app_uri,
+            app_version_uuid=self._app_version['public_id'],
             experiment_id=experiment_id,
             machine=machine,
             module_input_serialized=module_input_serialized,
@@ -86,6 +86,7 @@ class BioLibApp:
             override_command=override_command,
             result_prefix=result_prefix,
             timeout=timeout,
+            requested_machine_count=machine_count,
         )
         if blocking:
             # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
@@ -93,8 +94,8 @@ class BioLibApp:
                 utils.STREAM_STDOUT = True
 
             enable_print = bool(
-                utils.STREAM_STDOUT and
-                (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
+                utils.STREAM_STDOUT
+                and (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
             )
             job._stream_logs(enable_print=enable_print)  # pylint: disable=protected-access
 
@@ -108,11 +109,11 @@ class BioLibApp:
             self.cli()
 
         else:
-            raise BioLibError('''
+            raise BioLibError("""
 Calling an app directly with app() is currently being reworked.
 To use the previous functionality, please call app.cli() instead. 
 Example: "app.cli('--help')"
-''')
+""")
 
     @staticmethod
     def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
@@ -142,9 +143,9 @@ Example: "app.cli('--help')"
                     args[idx] = Path(arg).name
 
                 # support --myarg=file.txt
-                elif os.path.isfile(arg.split("=")[-1]) or os.path.isdir(arg.split("=")[-1]):
-                    files.append(arg.split("=")[-1])
-                    args[idx] = arg.split("=")[0] + '=' + Path(arg.split("=")[-1]).name
+                elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
+                    files.append(arg.split('=')[-1])
+                    args[idx] = arg.split('=')[0] + '=' + Path(arg.split('=')[-1]).name
                 else:
                     pass  # a normal string arg was given
             else:
@@ -154,7 +155,7 @@ Example: "app.cli('--help')"
                 elif isinstance(arg, io.BytesIO):
                     file_data = arg.getvalue()
                 else:
-                    raise Exception(f"Unexpected type of argument: {arg}")
+                    raise Exception(f'Unexpected type of argument: {arg}')
                 files_dict[f'/{tmp_filename}'] = file_data
                 args[idx] = tmp_filename
 
@@ -192,48 +193,10 @@ Example: "app.cli('--help')"
         )
         return module_input_serialized
 
-    def _start_in_cloud(
-            self,
-            module_input_serialized: bytes,
-            override_command: bool = False,
-            machine: Optional[str] = None,
-            experiment_id: Optional[str] = None,
-            result_prefix: Optional[str] = None,
-            timeout: Optional[int] = None,
-            notify: bool = False,
-    ) -> Job:
-        if len(module_input_serialized) < 500_000:
-            _job_dict = BiolibJobApi.create_job_with_data(
-                app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
-                app_version_uuid=self._app_version['public_id'],
-                arguments_override_command=override_command,
-                experiment_uuid=experiment_id,
-                module_input_serialized=module_input_serialized,
-                notify=notify,
-                requested_machine=machine,
-                requested_timeout_seconds=timeout,
-                result_name_prefix=result_prefix,
-            )
-            return Job(cast(JobDict, _job_dict))
-
-        job_dict: CreatedJobDict = BiolibJobApi.create(
-            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
-            app_version_id=self._app_version['public_id'],
-            experiment_uuid=experiment_id,
-            machine=machine,
-            notify=notify,
-            override_command=override_command,
-            timeout=timeout,
-        )
-        JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
-        cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
-        logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
-        return Job(cast(JobDict, job_dict))
-
     def _run_locally(self, module_input_serialized: bytes) -> Job:
         job_dict = BiolibJobApi.create(
             app_version_id=self._app_version['public_id'],
-            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix']
+            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
         )
         job = Job(job_dict)
 
@@ -263,7 +226,12 @@ Example: "app.cli('--help')"
             if not key.startswith('--'):
                 key = f'--{key}'
 
-            args.extend([key, value])
+            args.append(key)
+            if isinstance(value, list):
+                # TODO: only do this if argument key is of type file list
+                args.extend(value)
+            else:
+                args.append(value)
 
         return self.cli(args, **biolib_kwargs)