pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/app/app.py

@@ -1,29 +1,74 @@
+import copy
+import io
+import json
 import os
+import posixpath
+import random
+import string
+from pathlib import Path
 
-from biolib.app.utils import run_job
+from biolib import utils
+from biolib._internal.file_utils import path_to_renamed_path
+from biolib._runtime.runtime import Runtime
+from biolib._shared.utils import parse_resource_uri
+from biolib.api.client import ApiClient
 from biolib.biolib_api_client import JobState
 from biolib.biolib_api_client.app_types import App, AppVersion
-from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
-from biolib.app.app_result import AppResult
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import ModuleInput
-from biolib.biolib_errors import BioLibError
+from biolib.biolib_errors import BioLibError, JobResultNonZeroExitCodeError
 from biolib.biolib_logging import logger
+from biolib.compute_node.job_worker.job_worker import JobWorker
+from biolib.experiments.experiment import Experiment
+from biolib.jobs.job import Result
+from biolib.typing_utils import Dict, Optional
+
+
+class JsonStringIO(io.StringIO):
+    pass
 
 
 class BioLibApp:
+    def __init__(
+        self,
+        uri: str,
+        _api_client: Optional[ApiClient] = None,
+        suppress_version_warning: bool = False,
+        _experiment: Optional[str] = None,
+    ):
+        self._api_client: Optional[ApiClient] = _api_client
+        self._experiment = _experiment
+        self._input_uri = uri
+        self._parsed_input_uri = parse_resource_uri(uri)
 
-    def __init__(self, uri: str):
-        app_response = BiolibAppApi.get_by_uri(uri)
+        app_response = BiolibAppApi.get_by_uri(uri=uri, api_client=self._api_client)
         self._app: App = app_response['app']
         self._app_uri = app_response['app_uri']
         self._app_version: AppVersion = app_response['app_version']
 
-        logger.info(f'Loaded project {self._app_uri}')
+        if not suppress_version_warning:
+            if self._parsed_input_uri['version'] is None:
+                if Runtime.check_is_environment_biolib_app():
+                    logger.warning(
+                        f"No version specified in URI '{uri}'. This will use the default version, "
+                        f'which may change behaviour over time. Consider locking down the exact version, '
+                        f"e.g. '{uri}:1.2.3'"
+                    )
+
+        if self._parsed_input_uri['tag']:
+            semantic_version = f"{self._app_version['major']}.{self._app_version['minor']}.{self._app_version['patch']}"
+            logger.info(f'Loaded {self._input_uri} (resolved to {semantic_version})')
+        else:
+            logger.info(f'Loaded {self._app_uri}')
 
     def __str__(self) -> str:
         return self._app_uri
 
+    @property
+    def uri(self) -> str:
+        return self._app_uri
+
     @property
     def uuid(self) -> str:
         return self._app['public_id']
@@ -32,50 +77,98 @@ class BioLibApp:
     def version(self) -> AppVersion:
         return self._app_version
 
-    def cli(self, args=None, stdin=None, files=None):
+    def cli(
+        self,
+        args=None,
+        stdin=None,
+        files=None,
+        override_command=False,
+        machine='',
+        blocking: bool = True,
+        experiment_id: Optional[str] = None,
+        result_prefix: Optional[str] = None,
+        timeout: Optional[int] = None,
+        notify: bool = False,
+        max_workers: Optional[int] = None,
+        experiment: Optional[str] = None,
+        temporary_client_secrets: Optional[Dict[str, str]] = None,
+        check: bool = False,
+        stream_logs: bool = False,
+    ) -> Result:
+        if experiment_id and experiment:
+            raise ValueError('Only one of experiment_id and experiment can be specified')
+
+        if check and not blocking:
+            raise ValueError('The argument "check" cannot be True when blocking is False')
+
+        if not experiment_id:
+            experiment_to_use = experiment if experiment is not None else self._experiment
+            experiment_instance: Optional[Experiment]
+            if experiment_to_use:
+                experiment_instance = Experiment(experiment_to_use, _api_client=self._api_client)
+            else:
+                experiment_instance = Experiment.get_experiment_in_context()
+            experiment_id = experiment_instance.uuid if experiment_instance else None
+
         module_input_serialized = self._get_serialized_module_input(args, stdin, files)
 
-        job = BiolibJobApi.create(self._app_version['public_id'])
-        BiolibJobApi.update_state(job['public_id'], JobState.IN_PROGRESS.value)
+        if machine == 'local':
+            raise BioLibError('Running applications locally with machine="local" is no longer supported.')
 
-        try:
-            module_output = run_job(job, module_input_serialized)
-            try:
-                BiolibJobApi.update_state(job_id=job['public_id'], state=JobState.COMPLETED.value)
-            except Exception as error:  # pylint: disable=broad-except
-                logger.warning(f'Could not update job state to completed:\n{error}')
-
-            return AppResult(
-                exitcode=module_output['exit_code'],
-                stderr=module_output['stderr'],
-                stdout=module_output['stdout'],
-                files=module_output['files']
+        job = Result._start_job_in_cloud(  # pylint: disable=protected-access
+            app_uri=self._app_uri,
+            app_version_uuid=self._app_version['public_id'],
+            experiment_id=experiment_id,
+            machine=machine,
+            module_input_serialized=module_input_serialized,
+            notify=notify,
+            override_command=override_command,
+            result_prefix=result_prefix,
+            timeout=timeout,
+            requested_machine_count=max_workers,
+            temporary_client_secrets=temporary_client_secrets,
+            api_client=self._api_client,
+        )
+        if utils.IS_RUNNING_IN_NOTEBOOK:
+            logger.info(f'View the result in your browser at: {utils.BIOLIB_BASE_URL}/results/{job.id}/')
+        if blocking:
+            # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
+            if utils.IS_RUNNING_IN_NOTEBOOK:
+                utils.STREAM_STDOUT = True
+
+            enable_print = bool(
+                (utils.STREAM_STDOUT or stream_logs)
+                and (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
             )
+            job._stream_logs(enable_print=enable_print)  # pylint: disable=protected-access
+
+            if check:
+                exit_code = job.get_exit_code()
+                if exit_code != 0:
+                    raise JobResultNonZeroExitCodeError(exit_code)
 
-        except BioLibError as exception:
-            logger.error(f'Compute failed with: {exception.message}')
-            try:
-                BiolibJobApi.update_state(job_id=job['public_id'], state=JobState.FAILED.value)
-            except Exception as error:  # pylint: disable=broad-except
-                logger.warning(f'Could not update job state to failed:\n{error}')
+        return job
 
-            raise exception
+    def exec(self, args=None, stdin=None, files=None, machine=''):
+        return self.cli(args, stdin, files, override_command=True, machine=machine)
 
     def __call__(self, *args, **kwargs):
         if not args and not kwargs:
             self.cli()
 
         else:
-            raise BioLibError('''
+            raise BioLibError("""
 Calling an app directly with app() is currently being reworked.
-To use the previous functionality, please call app.cli() instead. 
+To use the previous functionality, please call app.cli() instead.
 Example: "app.cli('--help')"
-''')
+""")
 
     @staticmethod
     def _get_serialized_module_input(args=None, stdin=None, files=None) -> bytes:
         if args is None:
             args = []
+        else:
+            args = copy.copy(args)
 
         if stdin is None:
             stdin = b''
@@ -91,26 +184,136 @@ Example: "app.cli('--help')"
 
         if files is None:
             files = []
-            for idx, arg in enumerate(args):
-                if os.path.isfile(arg):
-                    files.append(arg)
-                    args[idx] = arg.split('/')[-1]
 
-        cwd = os.getcwd()
         files_dict = {}
+        if isinstance(files, list):
+            for file_path in files:
+                path = Path(file_path)
+                if path.is_dir():
+                    renamed_dir = path_to_renamed_path(file_path)
+                    for filename in path.rglob('*'):
+                        if filename.is_dir():
+                            continue
+                        with open(filename, 'rb') as f:
+                            relative_to_dir = filename.resolve().relative_to(path.resolve())
+                            files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
+                else:
+                    with open(path, 'rb') as f:
+                        files_dict[path_to_renamed_path(str(path))] = f.read()
+        elif isinstance(files, dict):
+            files_dict = {}
+            for key, value in files.items():
+                if '//' in key:
+                    raise BioLibError(f"File path '{key}' contains double slashes which are not allowed")
+                if not key.startswith('/'):
+                    key = '/' + key
+                files_dict[key] = value
+        else:
+            raise Exception('The given files input must be list or dict or None')
 
-        for file in files:
-            path = file
-            if not file.startswith('/'):
-                # make path absolute
-                path = cwd + '/' + file
-
-            arg_split = path.split('/')
-            file = open(path, 'rb')
-            path = '/' + arg_split[-1]
+        for idx, arg in enumerate(args):
+            if isinstance(arg, str):
+                if os.path.isfile(arg) or os.path.isdir(arg):
+                    if os.path.isfile(arg):
+                        with open(arg, 'rb') as f:
+                            files_dict[path_to_renamed_path(arg)] = f.read()
+                    elif os.path.isdir(arg):
+                        path = Path(arg)
+                        renamed_dir = path_to_renamed_path(arg)
+                        for filename in path.rglob('*'):
+                            if filename.is_dir():
+                                continue
+                            with open(filename, 'rb') as f:
+                                relative_to_dir = filename.resolve().relative_to(path.resolve())
+                                files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
+                    args[idx] = path_to_renamed_path(arg, prefix_with_slash=False)
 
-            files_dict[path] = file.read()
-            file.close()
+                # support --myarg=file.txt
+                elif os.path.isfile(arg.split('=')[-1]) or os.path.isdir(arg.split('=')[-1]):
+                    file_path = arg.split('=')[-1]
+                    if os.path.isfile(file_path):
+                        with open(file_path, 'rb') as f:
+                            files_dict[path_to_renamed_path(file_path)] = f.read()
+                    elif os.path.isdir(file_path):
+                        path = Path(file_path)
+                        renamed_dir = path_to_renamed_path(file_path)
+                        for filename in path.rglob('*'):
+                            if filename.is_dir():
+                                continue
+                            with open(filename, 'rb') as f:
+                                relative_to_dir = filename.resolve().relative_to(path.resolve())
+                                files_dict[posixpath.join(renamed_dir, relative_to_dir.as_posix())] = f.read()
+                    args[idx] = arg.split('=')[0] + '=' + path_to_renamed_path(file_path, prefix_with_slash=False)
+                else:
+                    pass  # a normal string arg was given
+            else:
+                tmp_filename = f'input_{"".join(random.choices(string.ascii_letters + string.digits, k=7))}'
+                if isinstance(arg, JsonStringIO):
+                    file_data = arg.getvalue().encode()
+                    tmp_filename += '.json'
+                elif isinstance(arg, io.StringIO):
+                    file_data = arg.getvalue().encode()
+                elif isinstance(arg, io.BytesIO):
+                    file_data = arg.getvalue()
+                else:
+                    raise Exception(f'Unexpected type of argument: {arg}')
+                files_dict[f'/{tmp_filename}'] = file_data
+                args[idx] = tmp_filename
 
-        module_input_serialized: bytes = ModuleInput().serialize(stdin=stdin, arguments=args, files=files_dict)
+        module_input_serialized: bytes = ModuleInput().serialize(
+            stdin=stdin,
+            arguments=args,
+            files=files_dict,
+        )
         return module_input_serialized
+
+    def _run_locally(self, module_input_serialized: bytes) -> Result:
+        job_dict = BiolibJobApi.create(
+            app_version_id=self._app_version['public_id'],
+            app_resource_name_prefix=parse_resource_uri(self._app_uri)['resource_prefix'],
+        )
+        job = Result(job_dict)
+
+        try:
+            BiolibJobApi.update_state(job.id, JobState.IN_PROGRESS)
+            module_output = JobWorker().run_job_locally(job_dict, module_input_serialized)
+            job._set_result_module_output(module_output)  # pylint: disable=protected-access
+            BiolibJobApi.update_state(job.id, JobState.COMPLETED)
+        except BaseException as error:
+            BiolibJobApi.update_state(job.id, JobState.FAILED)
+            raise error
+
+        return job
+
+    def run(self, **kwargs) -> Result:
+        args = []
+        biolib_kwargs = {}
+        for key, value in kwargs.items():
+            if key.startswith('biolib_'):
+                biolib_kwarg_key = key.replace('biolib_', '')
+                biolib_kwargs[biolib_kwarg_key] = value
+                continue
+
+            if isinstance(value, dict):
+                value = JsonStringIO(json.dumps(value))
+            elif isinstance(value, (int, float)):  # Cast numeric values to strings
+                value = str(value)
+
+            if not key.startswith('--'):
+                key = f'--{key}'
+
+            args.append(key)
+            if isinstance(value, list):
+                # TODO: only do this if argument key is of type file list
+                args.extend(value)
+            else:
+                args.append(value)
+
+        # Set check=True by default if not explicitly provided and not in non-blocking mode
+        if 'check' not in biolib_kwargs and biolib_kwargs.get('blocking', True) is not False:
+            biolib_kwargs['check'] = True
+
+        return self.cli(args, **biolib_kwargs)
+
+    def start(self, **kwargs) -> Result:
+        return self.run(biolib_blocking=False, **kwargs)

biolib/app/search_apps.py

@@ -0,0 +1,45 @@
+# TODO: Fix ignore of type
+# type: ignore
+from biolib import api, utils
+from biolib.typing_utils import Optional, List
+
+
+def search_apps(
+        search_query: Optional[str] = None,
+        team: Optional[str] = None,
+        count: int = 100,
+) -> List[str]:
+    query_exceeded_page_size = False
+    params = {
+        'page_size': count,
+    }
+    if team:
+        if not team.startswith('@'):
+            team = '@biolib.com/' + team
+        params['account_handle'] = team
+
+    if search_query:
+        params['search'] = search_query
+
+    apps_json = api.client.get(path='/apps/', params=params).json()
+    if apps_json['count'] > count:
+        query_exceeded_page_size = True
+
+    apps = [app['resource_uri'] for app in apps_json['results']]
+
+    if not utils.BASE_URL_IS_PUBLIC_BIOLIB and (not team or team.lower().startswith('@biolib.com')):
+        # Also get federated apps if running on enterprise deployment
+        public_biolib_apps_json = api.client.get(
+            authenticate=False,
+            path='https://biolib.com/api/apps/',
+            params=params,
+        ).json()
+        if public_biolib_apps_json['count'] > count:
+            query_exceeded_page_size = True
+
+        apps.extend([f"@biolib.com/{app['resource_uri']}" for app in public_biolib_apps_json['results']])
+
+    if query_exceeded_page_size:
+        print(f'Search results exceeded {count}, use the argument "count" to increase the amount of results returned')
+
+    return apps

biolib/biolib_api_client/api_client.py

@@ -1,46 +1,130 @@
 import os
+from datetime import datetime, timezone
 from json.decoder import JSONDecodeError
 
-import requests
-from biolib.typing_utils import Optional
+from biolib._internal.http_client import HttpClient
+from biolib._internal.utils.auth import decode_jwt_without_checking_signature
+from biolib._runtime.runtime import Runtime
 from biolib.biolib_errors import BioLibError
-from biolib.biolib_logging import logger
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.typing_utils import Optional, TypedDict
+
+from .user_state import UserState
+
+
+class UserTokens(TypedDict):
+    access: str
+    refresh: str
 
 
 class _ApiClient:
     def __init__(self, base_url: str, access_token: Optional[str] = None):
         self.base_url: str = base_url
+        self.access_token: Optional[str] = access_token  # TODO: Deprecate passing access_token in constructor
         self.refresh_token: Optional[str] = None
-        self.access_token: Optional[str] = access_token
+        self.resource_deploy_key: Optional[str] = None
+
+        self._user_state = UserState()
+        self._sign_in_attempted: bool = False
 
     @property
     def is_signed_in(self) -> bool:
-        return self.refresh_token is not None and self.access_token is not None
+        return bool(self.refresh_token or self.resource_deploy_key)
 
-    def login(self, api_token: Optional[str], exit_on_failure=False):
-        if not api_token:
-            if exit_on_failure:
-                raise BioLibError('Error: Attempted login, but BIOLIB_TOKEN was not set, exiting...')
-            else:
-                logger.debug('Attempted login, but BIOLIB_TOKEN was not set, so continuing without logging in')
+    def set_user_tokens(self, user_tokens: UserTokens) -> None:
+        with self._user_state as user_state:
+            user_state['refresh_token'] = user_tokens['refresh']
+
+        self.access_token = user_tokens['access']
+        self.refresh_token = user_tokens['refresh']
+
+    def sign_out(self) -> None:
+        api_token = os.getenv('BIOLIB_TOKEN', default=None)
+        if api_token:
+            print('To sign out unset the environment variable "BIOLIB_TOKEN"')
+
+        self.access_token = None
+        self.refresh_token = None
+
+        with self._user_state as user_state:
+            user_state['refresh_token'] = None
+
+    def refresh_access_token(self) -> None:
+        if not self.is_signed_in or self.resource_deploy_key:
+            return
+
+        if self.access_token:
+            decoded_token = decode_jwt_without_checking_signature(self.access_token)
+            if datetime.now(tz=timezone.utc).timestamp() < decoded_token['payload']['exp'] - 60:  # 60 second buffer
+                # Token has not expired yet
                 return
 
-        response = requests.post(
-            f'{self.base_url}/api/user/api_tokens/exchange/',
-            json={'token': api_token},
-        )
+        # TODO: Implement nicer error handling
         try:
-            json_response = response.json()
+            response = HttpClient.request(
+                method='POST',
+                url=f'{self.base_url}/api/user/token/refresh/',
+                data={'refresh': self.refresh_token},
+            )
+        except Exception as exception:
+            logger.error('Sign in with refresh token failed')
+            raise exception
+
+        try:
+            response_dict = response.json()
         except JSONDecodeError as error:
             logger.error('Could not decode response from server as JSON:')
             raise BioLibError(response.text) from error
-        if not response.ok:
-            logger.error('Login with API token failed:')
-            raise BioLibError(json_response['detail'])
+
+        self.access_token = response_dict['access']
+
+    def attempt_sign_in(self) -> None:
+        if not self._sign_in_attempted:
+            self._attempt_sign_in()
+            self._sign_in_attempted = True
+
+    def _attempt_sign_in(self) -> None:
+        api_token = os.getenv('BIOLIB_TOKEN', default=None)
+
+        if api_token:
+            if api_token.startswith('bld_'):
+                self.resource_deploy_key = api_token
+            else:
+                self.sign_in_with_api_token(api_token)
         else:
-            self.refresh_token = json_response['refresh_token']
-            self.access_token = json_response['access_token']
-            logger.info('Successfully authenticated')
+            with self._user_state as user_state:
+                refresh_token_from_state = user_state['refresh_token']
+
+            # TODO: Handle expired refresh token
+            if refresh_token_from_state:
+                logger_no_user_data.debug('ApiClient: Signing in with refresh token from user state...')
+                self.refresh_token = refresh_token_from_state
+                try:
+                    self.refresh_access_token()
+                except Exception:  # pylint: disable=broad-except
+                    self.refresh_token = None
+                    with self._user_state as user_state:
+                        user_state['refresh_token'] = None
+
+    def sign_in_with_api_token(self, api_token: str) -> None:
+        logger_no_user_data.debug('ApiClient: Signing in with BIOLIB_TOKEN...')
+        try:
+            response = HttpClient.request(
+                method='POST',
+                url=f'{self.base_url}/api/user/api_tokens/exchange/',
+                data={'token': api_token},
+            )
+        except Exception as exception:
+            logger.error('Sign in with API token failed')
+            raise exception
+        try:
+            json_response = response.json()
+        except JSONDecodeError as error:
+            logger.error('Could not decode response from server as JSON')
+            raise BioLibError(response.text) from error
+
+        self.access_token = json_response['access_token']
+        self.refresh_token = json_response['refresh_token']
 
 
 class BiolibApiClient:
@@ -48,23 +132,34 @@ class BiolibApiClient:
 
     @staticmethod
     def initialize(base_url: str, access_token: Optional[str] = None):
-        BiolibApiClient.api_client = _ApiClient(base_url, access_token)
+        BiolibApiClient.api_client = _ApiClient(base_url=base_url, access_token=access_token)
 
     @staticmethod
-    def get() -> _ApiClient:
+    def get(attempt_sign_in: bool = True) -> _ApiClient:
         api_client = BiolibApiClient.api_client
-        if api_client is not None:
-            biolib_token = os.getenv('BIOLIB_TOKEN', default=None)
-            if biolib_token is not None and not api_client.is_signed_in:
-                api_client.login(api_token=biolib_token, exit_on_failure=True)
+        if api_client:
+            if attempt_sign_in:
+                api_client.attempt_sign_in()
             return api_client
+
+        raise BioLibError('Attempted to use uninitialized API client')
+
+    @staticmethod
+    def refresh_auth_token():
+        api_client = BiolibApiClient.get()
+        api_client.refresh_access_token()
+
+    @staticmethod
+    def is_reauthentication_needed() -> bool:
+        api_client = BiolibApiClient.get()
+        if not api_client.is_signed_in and not Runtime.check_is_environment_biolib_app():
+            return True
         else:
-            raise BioLibError('Attempted to use uninitialized API client')
+            return False
 
     @staticmethod
     def assert_is_signed_in(authenticated_action_description: str) -> None:
-        api_client = BiolibApiClient.get()
-        if not api_client.is_signed_in:
+        if BiolibApiClient.is_reauthentication_needed():
             raise BioLibError(
                 f'You must be signed in to {authenticated_action_description}. '
                 f'Please set the environment variable "BIOLIB_TOKEN"'

biolib/biolib_api_client/app_types.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
-from biolib.typing_utils import TypedDict, List, Optional, Dict, Literal
 from biolib.biolib_api_client.common_types import SemanticVersion
+from biolib.typing_utils import Dict, List, Literal, Optional, TypedDict
 
 
 class AppVersionSlim(SemanticVersion):
@@ -14,6 +14,9 @@ class AppVersion(AppVersionSlim):
     description: str
     is_runnable_by_user: bool
     source_code_license: str
+    stdout_render_type: Literal['text', 'markdown']
+    main_output_file: Optional[str]
+    app_uri: str
 
 
 class App(TypedDict):
@@ -28,6 +31,8 @@ class App(TypedDict):
     name: str
     public_id: str
     state: str
+    resource_uri: str
+    type: str
 
 
 class AppGetResponse(TypedDict):
@@ -57,34 +62,49 @@ class FilesMapping(TypedDict):
 
 
 class LargeFileSystemMapping(TypedDict):
-    uuid: str
+    presigned_download_url: str
     size_bytes: int
     to_path: str
+    uuid: str
 
 
-class Module(TypedDict):
+class _Module(TypedDict):
     command: str
     environment: Literal['biolib-app', 'biolib-custom', 'biolib-ecr']
     image_uri: str
+    absolute_image_uri: str
     estimated_image_size_bytes: Optional[int]
     input_files_mappings: List[FilesMapping]
     large_file_systems: List[LargeFileSystemMapping]
     name: str
     output_files_mappings: List[FilesMapping]
+    ports: List[int]
     source_files_mappings: List[FilesMapping]
     working_directory: str
 
 
+# type optional keys with total=False
+class Module(_Module, total=False):
+    secrets: Dict[str, str]
+
+
 class _AppVersionOnJob(TypedDict):
+    created_at: str
     client_side_executable_zip: Optional[str]
-    consumes_stdin: bool
     is_runnable_by_user: bool
     public_id: str
     remote_hosts: List[RemoteHost]
     settings: List[Dict]
     stdout_render_type: Literal['text', 'markdown']
+    main_output_file: Optional[str]
+
+
+class AppOnJob(TypedDict):
+    allow_client_side_execution: bool
+    state: Literal['public', 'draft']
 
 
 # type optional keys with total=False
 class AppVersionOnJob(_AppVersionOnJob, total=False):
+    app: AppOnJob
     modules: List[Module]