pybiolib 1.1.1881__py3-none-any.whl → 1.1.2193__py3-none-any.whl

biolib/_internal/push_application.py

@@ -6,12 +6,12 @@ import rich.progress
 import yaml
 
 from biolib import api, utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
 from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
-from biolib.lfs.utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.typing_utils import Iterable, Optional, Set, TypedDict
 
 REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')

biolib/_internal/runtime.py

@@ -1,7 +1,4 @@
-import json
-
-from biolib import api
-from biolib.typing_utils import Optional, TypedDict, cast
+from biolib.typing_utils import TypedDict
 
 
 class RuntimeJobDataDict(TypedDict):
@@ -9,6 +6,7 @@ class RuntimeJobDataDict(TypedDict):
     job_requested_machine: str
     job_uuid: str
     job_auth_token: str
+    app_uri: str
 
 
 class BioLibRuntimeError(Exception):
@@ -19,55 +17,3 @@ class BioLibRuntimeNotRecognizedError(BioLibRuntimeError):
     def __init__(self, message='The runtime is not recognized as a BioLib app'):
         self.message = message
         super().__init__(self.message)
-
-
-class Runtime:
-    _job_data: Optional[RuntimeJobDataDict] = None
-
-    @staticmethod
-    def check_is_environment_biolib_app() -> bool:
-        return bool(Runtime._try_to_get_job_data())
-
-    @staticmethod
-    def get_job_id() -> str:
-        return Runtime._get_job_data()['job_uuid']
-
-    @staticmethod
-    def get_job_auth_token() -> str:
-        return Runtime._get_job_data()['job_auth_token']
-
-    @staticmethod
-    def get_job_requested_machine() -> str:
-        return Runtime._get_job_data()['job_requested_machine']
-
-    @staticmethod
-    def set_main_result_prefix(result_prefix: str) -> None:
-        job_data = Runtime._get_job_data()
-        api.client.patch(
-            data={'result_name_prefix': result_prefix},
-            headers={'Job-Auth-Token': job_data['job_auth_token']},
-            path=f"/jobs/{job_data['job_uuid']}/main_result/",
-        )
-
-    @staticmethod
-    def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
-        if not Runtime._job_data:
-            try:
-                with open('/biolib/secrets/biolib_system_secret') as file:
-                    job_data: RuntimeJobDataDict = json.load(file)
-            except BaseException:
-                return None
-
-            if not job_data['version'].startswith('1.'):
-                raise BioLibRuntimeError(f"Unexpected system secret version {job_data['version']} expected 1.x.x")
-
-            Runtime._job_data = job_data
-
-        return cast(RuntimeJobDataDict, Runtime._job_data)
-
-    @staticmethod
-    def _get_job_data() -> RuntimeJobDataDict:
-        job_data = Runtime._try_to_get_job_data()
-        if not job_data:
-            raise BioLibRuntimeNotRecognizedError() from None
-        return job_data

biolib/_internal/types/__init__.py

@@ -0,0 +1,4 @@
+from .app import *  # noqa: F403
+from .data_record import *  # noqa: F403
+from .experiment import *  # noqa: F403
+from .resource import *  # noqa: F403

biolib/_internal/types/app.py

@@ -0,0 +1,9 @@
+from .typing import TypedDict
+
+
+class AppSlimDict(TypedDict):
+    pass
+
+
+class AppDetailedDict(AppSlimDict):
+    pass

biolib/_internal/types/data_record.py

@@ -0,0 +1,40 @@
+from .typing import Dict, List, Literal, Optional, TypedDict, Union
+
+
+class SqliteV1ForeignKey(TypedDict):
+    table: str
+    column: str
+
+
+class SqliteV1Column(TypedDict):
+    type: Literal['INTEGER', 'REAL', 'TEXT', 'JSON']  # noqa:F821
+    nullable: Optional[bool]
+    foreign_key: Optional[SqliteV1ForeignKey]
+    json_schema: Optional[Dict]
+
+
+class SqliteV1Table(TypedDict):
+    columns: Dict[str, SqliteV1Column]
+
+
+class SqliteV1DatabaseSchema(TypedDict):
+    tables: Dict[str, SqliteV1Table]
+
+
+class DataRecordValidationRuleDict(TypedDict):
+    path: str
+    type: str
+    rule: Union[SqliteV1DatabaseSchema]
+
+
+class DataRecordTypeDict(TypedDict):
+    name: str
+    validation_rules: List[DataRecordValidationRuleDict]
+
+
+class DataRecordSlimDict(TypedDict):
+    pass
+
+
+class DataRecordDetailedDict(DataRecordSlimDict):
+    type: Optional[DataRecordTypeDict]

biolib/_internal/types/experiment.py

@@ -0,0 +1,10 @@
+from .typing import TypedDict
+
+
+class ExperimentSlimDict(TypedDict):
+    job_count: int
+    job_running_count: int
+
+
+class ExperimentDetailedDict(ExperimentSlimDict):
+    pass

biolib/_internal/types/resource.py

@@ -0,0 +1,14 @@
+from .app import AppSlimDict
+from .data_record import DataRecordSlimDict
+from .experiment import ExperimentSlimDict
+from .typing import Optional, TypedDict
+
+
+class ResourceDict(TypedDict):
+    uuid: str
+    uri: str
+    name: str
+    created_at: str
+    app: Optional[AppSlimDict]
+    data_record: Optional[DataRecordSlimDict]
+    experiment: Optional[ExperimentSlimDict]

biolib/_internal/types/typing.py

@@ -0,0 +1,7 @@
+import sys
+
+# import and expose everything from the typing module
+from typing import *  # noqa:F403 pylint: disable=wildcard-import, unused-wildcard-import
+
+if sys.version_info < (3, 8):  # noqa: UP036
+    from typing_extensions import Literal, TypedDict  # pylint: disable=unused-import

biolib/_runtime/runtime.py

@@ -0,0 +1,80 @@
+import json
+import re
+from typing import Optional
+
+from biolib import api
+from biolib._internal.runtime import BioLibRuntimeError, BioLibRuntimeNotRecognizedError, RuntimeJobDataDict
+from biolib.typing_utils import cast
+
+
+class Runtime:
+    _job_data: Optional[RuntimeJobDataDict] = None
+
+    @staticmethod
+    def check_is_environment_biolib_app() -> bool:
+        return bool(Runtime._try_to_get_job_data())
+
+    @staticmethod
+    def get_job_id() -> str:
+        return Runtime._get_job_data()['job_uuid']
+
+    @staticmethod
+    def get_job_auth_token() -> str:
+        return Runtime._get_job_data()['job_auth_token']
+
+    @staticmethod
+    def get_job_requested_machine() -> str:
+        return Runtime._get_job_data()['job_requested_machine']
+
+    @staticmethod
+    def get_app_uri() -> str:
+        return Runtime._get_job_data()['app_uri']
+
+    @staticmethod
+    def get_secret(secret_name: str) -> bytes:
+        assert re.match(
+            '^[a-zA-Z0-9_-]*$', secret_name
+        ), 'Secret name can only contain alphanumeric characters and dashes or underscores '
+        try:
+            with open(f'/biolib/secrets/{secret_name}', 'rb') as file:
+                return file.read()
+        except BaseException as error:
+            raise BioLibRuntimeError(f'Unable to get system secret: {secret_name}') from error
+
+    @staticmethod
+    def set_main_result_prefix(result_prefix: str) -> None:
+        job_data = Runtime._get_job_data()
+        api.client.patch(
+            data={'result_name_prefix': result_prefix},
+            headers={'Job-Auth-Token': job_data['job_auth_token']},
+            path=f"/jobs/{job_data['job_uuid']}/main_result/",
+        )
+
+    @staticmethod
+    def create_result_note(note: str) -> None:
+        job_id = Runtime.get_job_id()
+        # Note: Authentication is added by app caller proxy in compute node
+        api.client.post(data={'note': note}, path=f'/jobs/{job_id}/notes/')
+
+    @staticmethod
+    def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
+        if not Runtime._job_data:
+            try:
+                with open('/biolib/secrets/biolib_system_secret') as file:
+                    job_data: RuntimeJobDataDict = json.load(file)
+            except BaseException:
+                return None
+
+            if not job_data['version'].startswith('1.'):
+                raise BioLibRuntimeError(f"Unexpected system secret version {job_data['version']} expected 1.x.x")
+
+            Runtime._job_data = job_data
+
+        return cast(RuntimeJobDataDict, Runtime._job_data)
+
+    @staticmethod
+    def _get_job_data() -> RuntimeJobDataDict:
+        job_data = Runtime._try_to_get_job_data()
+        if not job_data:
+            raise BioLibRuntimeNotRecognizedError() from None
+        return job_data

biolib/api/__init__.py

@@ -1,4 +1,5 @@
 from biolib._internal.http_client import HttpClient
+
 from .client import ApiClient as _ApiClient
 
 _client = _ApiClient()

biolib/api/client.py

@@ -1,8 +1,10 @@
-from urllib.parse import urljoin, urlencode
+from urllib.parse import urlencode, urljoin
 
-from biolib.typing_utils import Dict, Optional, Union
+import importlib_metadata
+
+from biolib._internal.http_client import HttpClient, HttpResponse
 from biolib.biolib_api_client import BiolibApiClient as DeprecatedApiClient
-from biolib._internal.http_client import HttpResponse, HttpClient
+from biolib.typing_utils import Dict, Optional, Union, cast
 
 OptionalHeaders = Union[
     Optional[Dict[str, str]],
@@ -10,14 +12,24 @@ OptionalHeaders = Union[
 ]
 
 
+def _get_biolib_package_version() -> str:
+    # try fetching version, if it fails (usually when in dev), add default
+    try:
+        return cast(str, importlib_metadata.version('pybiolib'))
+    except importlib_metadata.PackageNotFoundError:
+        return '0.0.0'
+
+
 class ApiClient(HttpClient):
+    _biolib_package_version: str = _get_biolib_package_version()
+
     def get(
-            self,
-            path: str,
-            params: Optional[Dict[str, Union[str, int]]] = None,
-            headers: OptionalHeaders = None,
-            authenticate: bool = True,
-            retries: int = 10,
+        self,
+        path: str,
+        params: Optional[Dict[str, Union[str, int]]] = None,
+        headers: OptionalHeaders = None,
+        authenticate: bool = True,
+        retries: int = 10,
     ) -> HttpResponse:
         return self.request(
             headers=self._get_headers(opt_headers=headers, authenticate=authenticate),
@@ -27,12 +39,12 @@ class ApiClient(HttpClient):
         )
 
     def post(
-            self,
-            path: str,
-            data: Optional[Union[Dict, bytes]] = None,
-            headers: OptionalHeaders = None,
-            authenticate: bool = True,
-            retries: int = 5,
+        self,
+        path: str,
+        data: Optional[Union[Dict, bytes]] = None,
+        headers: OptionalHeaders = None,
+        authenticate: bool = True,
+        retries: int = 50,  # TODO: reduce this back to 5 when timeout errors have been solved
     ) -> HttpResponse:
         return self.request(
             data=data,
@@ -42,13 +54,20 @@ class ApiClient(HttpClient):
             url=self._get_absolute_url(path=path, query_params=None),
         )
 
-    def patch(self, path: str, data: Dict, headers: OptionalHeaders = None, retries: int = 5) -> HttpResponse:
+    def patch(
+        self,
+        path: str,
+        data: Dict,
+        headers: OptionalHeaders = None,
+        retries: int = 5,
+        params: Optional[Dict[str, Union[str, int]]] = None,
+    ) -> HttpResponse:
         return self.request(
             data=data,
             headers=self._get_headers(opt_headers=headers, authenticate=True),
             method='PATCH',
             retries=retries,
-            url=self._get_absolute_url(path=path, query_params=None),
+            url=self._get_absolute_url(path=path, query_params=params),
         )
 
     @staticmethod
@@ -67,6 +86,9 @@ class ApiClient(HttpClient):
         if access_token and authenticate:
             headers['Authorization'] = f'Bearer {access_token}'
 
+        headers['client-type'] = 'biolib-python'
+        headers['client-version'] = ApiClient._biolib_package_version
+
         return headers
 
     @staticmethod

biolib/app/app.py

@@ -1,29 +1,26 @@
-import os
 import io
-import random
 import json
+import os
+import random
 import string
-
 from pathlib import Path
+
 from biolib import utils
-from biolib.compute_node.job_worker.job_storage import JobStorage
-from biolib.compute_node.job_worker.job_worker import JobWorker
-from biolib.experiments.experiment import Experiment
-from biolib.jobs import Job
-from biolib.typing_utils import Optional, cast
-from biolib.biolib_api_client import CreatedJobDict, JobState
-from biolib.jobs.types import JobDict
+from biolib.biolib_api_client import JobState
 from biolib.biolib_api_client.app_types import App, AppVersion
-from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import ModuleInput
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
+from biolib.compute_node.job_worker.job_worker import JobWorker
+from biolib.experiments.experiment import Experiment
+from biolib.jobs import Job
+from biolib.typing_utils import Optional
 from biolib.utils.app_uri import parse_app_uri
 
 
 class BioLibApp:
-
     def __init__(self, uri: str):
         app_response = BiolibAppApi.get_by_uri(uri)
         self._app: App = app_response['app']
@@ -48,17 +45,18 @@ class BioLibApp:
         return self._app_version
 
     def cli(
-            self,
-            args=None,
-            stdin=None,
-            files=None,
-            override_command=False,
-            machine='',
-            blocking: bool = True,
-            experiment_id: Optional[str] = None,
-            result_prefix: Optional[str] = None,
-            timeout: Optional[int] = None,
-            notify: bool = False,
+        self,
+        args=None,
+        stdin=None,
+        files=None,
+        override_command=False,
+        machine='',
+        blocking: bool = True,
+        experiment_id: Optional[str] = None,
+        result_prefix: Optional[str] = None,
+        timeout: Optional[int] = None,
+        notify: bool = False,
+        machine_count: Optional[int] = None,
     ) -> Job:
         if not experiment_id:
             experiment = Experiment.get_experiment_in_context()
@@ -78,7 +76,9 @@ class BioLibApp:
 
             return self._run_locally(module_input_serialized)
 
-        job = self._start_in_cloud(
+        job = Job._start_job_in_cloud(  # pylint: disable=protected-access
+            app_uri=self._app_uri,
+            app_version_uuid=self._app_version['public_id'],
             experiment_id=experiment_id,
             machine=machine,
             module_input_serialized=module_input_serialized,
@@ -86,6 +86,7 @@ class BioLibApp:
             override_command=override_command,
             result_prefix=result_prefix,
             timeout=timeout,
+            requested_machine_count=machine_count,
         )
         if blocking:
             # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
@@ -93,8 +94,8 @@ class BioLibApp:
                 utils.STREAM_STDOUT = True
 
             enable_print = bool(
-                utils.STREAM_STDOUT and
-                (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
+                utils.STREAM_STDOUT
+                and (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
             )
             job._stream_logs(enable_print=enable_print)  # pylint: disable=protected-access
 
@@ -108,11 +109,11 @@ class BioLibApp:
             self.cli()
 
         else:
-            raise BioLibError('''
+            raise BioLibError("""
 Calling an app directly with app() is currently being reworked.
 To use the previous functionality, please call app.cli() instead. 
 Example: "app.cli('--help')"
-''')
+""")
 
     @staticmethod
     def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
@@ -142,9 +143,9 @@ Example: "app.cli('--help')"
                     args[idx] = Path(arg).name
 
                 # support --myarg=file.txt
-                elif os.path.isfile(arg.split("=")[-1]) or os.path.isdir(arg.split("=")[-1]):
-                    files.append(arg.split("=")[-1])
-                    args[idx] = arg.split("=")[0] + '=' + Path(arg.split("=")[-1]).name
+                elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
+                    files.append(arg.split('=')[-1])
+                    args[idx] = arg.split('=')[0] + '=' + Path(arg.split('=')[-1]).name
                 else:
                     pass  # a normal string arg was given
             else:
@@ -154,7 +155,7 @@ Example: "app.cli('--help')"
                 elif isinstance(arg, io.BytesIO):
                     file_data = arg.getvalue()
                 else:
-                    raise Exception(f"Unexpected type of argument: {arg}")
+                    raise Exception(f'Unexpected type of argument: {arg}')
                 files_dict[f'/{tmp_filename}'] = file_data
                 args[idx] = tmp_filename
 
@@ -192,48 +193,10 @@ Example: "app.cli('--help')"
         )
         return module_input_serialized
 
-    def _start_in_cloud(
-            self,
-            module_input_serialized: bytes,
-            override_command: bool = False,
-            machine: Optional[str] = None,
-            experiment_id: Optional[str] = None,
-            result_prefix: Optional[str] = None,
-            timeout: Optional[int] = None,
-            notify: bool = False,
-    ) -> Job:
-        if len(module_input_serialized) < 500_000:
-            _job_dict = BiolibJobApi.create_job_with_data(
-                app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
-                app_version_uuid=self._app_version['public_id'],
-                arguments_override_command=override_command,
-                experiment_uuid=experiment_id,
-                module_input_serialized=module_input_serialized,
-                notify=notify,
-                requested_machine=machine,
-                requested_timeout_seconds=timeout,
-                result_name_prefix=result_prefix,
-            )
-            return Job(cast(JobDict, _job_dict))
-
-        job_dict: CreatedJobDict = BiolibJobApi.create(
-            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
-            app_version_id=self._app_version['public_id'],
-            experiment_uuid=experiment_id,
-            machine=machine,
-            notify=notify,
-            override_command=override_command,
-            timeout=timeout,
-        )
-        JobStorage.upload_module_input(job=job_dict, module_input_serialized=module_input_serialized)
-        cloud_job = BiolibJobApi.create_cloud_job(job_id=job_dict['public_id'], result_name_prefix=result_prefix)
-        logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
-        return Job(cast(JobDict, job_dict))
-
     def _run_locally(self, module_input_serialized: bytes) -> Job:
         job_dict = BiolibJobApi.create(
             app_version_id=self._app_version['public_id'],
-            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix']
+            app_resource_name_prefix=parse_app_uri(self._app_uri)['resource_name_prefix'],
         )
         job = Job(job_dict)
 

biolib/biolib_api_client/api_client.py

@@ -6,7 +6,7 @@ import os
 from datetime import datetime, timezone
 from json.decoder import JSONDecodeError
 
-from biolib._internal.runtime import Runtime
+from biolib._runtime.runtime import Runtime
 from biolib._internal.http_client import HttpClient
 from biolib.typing_utils import Optional
 from biolib.biolib_errors import BioLibError
@@ -192,9 +192,16 @@ class BiolibApiClient:
         api_client.refresh_access_token()
 
     @staticmethod
-    def assert_is_signed_in(authenticated_action_description: str) -> None:
+    def is_reauthentication_needed() -> bool:
         api_client = BiolibApiClient.get()
         if not api_client.is_signed_in and not Runtime.check_is_environment_biolib_app():
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def assert_is_signed_in(authenticated_action_description: str) -> None:
+        if BiolibApiClient.is_reauthentication_needed():
             raise BioLibError(
                 f'You must be signed in to {authenticated_action_description}. '
                 f'Please set the environment variable "BIOLIB_TOKEN"'

biolib/biolib_api_client/app_types.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
-from biolib.typing_utils import TypedDict, List, Optional, Dict, Literal
 from biolib.biolib_api_client.common_types import SemanticVersion
+from biolib.typing_utils import Dict, List, Literal, Optional, TypedDict
 
 
 class AppVersionSlim(SemanticVersion):
@@ -16,6 +16,7 @@ class AppVersion(AppVersionSlim):
     source_code_license: str
     stdout_render_type: Literal['text', 'markdown']
     main_output_file: Optional[str]
+    app_uri: str
 
 
 class App(TypedDict):
@@ -99,7 +100,6 @@ class _AppVersionOnJob(TypedDict):
 
 class AppOnJob(TypedDict):
     allow_client_side_execution: bool
-    can_push_data_record_for_user: bool
     state: Literal['public', 'draft']
 
 

biolib/biolib_api_client/biolib_job_api.py

@@ -46,6 +46,7 @@ class BiolibJobApi:
             experiment_uuid: Optional[str] = None,
             timeout: Optional[int] = None,
             notify: bool = False,
+            requested_machine_count: Optional[int] = None,
     ):
         data = {
             'app_version_id': app_version_id,
@@ -73,6 +74,9 @@ class BiolibJobApi:
                 'requested_machine': machine
             })
 
+        if requested_machine_count:
+            data.update({'requested_machine_count': requested_machine_count})
+
         if experiment_uuid:
             data['experiment_uuid'] = experiment_uuid
 
@@ -156,6 +160,7 @@ class BiolibJobApi:
             caller_job_uuid: Optional[str] = None,
             requested_timeout_seconds: Optional[int] = None,
             notify: bool = False,
+            requested_machine_count: Optional[int] = None,
     ) -> Dict:
         job_dict: Dict = biolib.api.client.post(
             path='/jobs/create_job_with_data/',
@@ -171,6 +176,7 @@ class BiolibJobApi:
                 'client-version': BIOLIB_PACKAGE_VERSION,
                 'experiment-uuid': experiment_uuid,
                 'requested-machine': requested_machine,
+                'requested-machine-count': str(requested_machine_count) if requested_machine_count else None,
                 'result-name-prefix': result_name_prefix,
                 'requested-timeout-seconds': str(requested_timeout_seconds) if requested_timeout_seconds else None,
                 'notify': 'true' if notify else 'false',

biolib/biolib_api_client/job_types.py

@@ -1,9 +1,8 @@
 from enum import Enum
 
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Optional, List
-
 from biolib.biolib_api_client.app_types import AppVersionOnJob, RemoteHost
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import List, Optional, TypedDict
 
 
 class JobState(Enum):
@@ -15,6 +14,7 @@ class JobState(Enum):
 
 
 class _Job(TypedDict):
+    app_uri: str
     app_version: AppVersionOnJob
     arguments_override_command: bool
     auth_token: str
@@ -22,10 +22,10 @@ class _Job(TypedDict):
     created_at: str
     federated_job_uuid: Optional[str]
     public_id: str
-    uuid: str
     remote_hosts_with_warning: List[RemoteHost]
     state: str
     user_id: Optional[str]
+    uuid: str
 
 
 # type optional keys with total=False