pybiolib 1.1.1151__py3-none-any.whl → 1.1.1172__py3-none-any.whl

biolib/app/app.py

@@ -4,12 +4,14 @@ import io
 import random
 import json
 import string
+
+from biolib import utils
 from biolib.compute_node.job_worker.job_storage import JobStorage
+from biolib.compute_node.job_worker.job_worker import JobWorker
 from biolib.experiments.experiment import Experiment
 from biolib.jobs import Job
-from biolib.typing_utils import List, Optional, cast
-from biolib.app.utils import run_job
-from biolib.biolib_api_client import JobState, BiolibApiClient, CreatedJobDict
+from biolib.typing_utils import Optional, cast
+from biolib.biolib_api_client import CreatedJobDict, JobState
 from biolib.jobs.types import JobDict
 from biolib.biolib_api_client.app_types import App, AppVersion
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
@@ -51,60 +53,43 @@ class BioLibApp:
             experiment_id: Optional[str] = None,
             result_prefix: Optional[str] = None,
     ) -> Job:
-
         if not experiment_id:
             experiment = Experiment.get_experiment_in_context()
             experiment_id = experiment.uuid if experiment else None
 
-        if not blocking:
-            return self._cli_non_blocking(
-                args,
-                stdin,
-                files,
-                override_command,
-                machine,
-                experiment_id,
-                result_prefix,
-            )
-
         module_input_serialized = self._get_serialized_module_input(args, stdin, files)
 
-        job: CreatedJobDict = BiolibJobApi.create(
-            app_version_id=self._app_version['public_id'],
-            override_command=override_command,
-            experiment_uuid=experiment_id,
-            machine=machine if machine != 'local' else '',
-        )
+        if machine == 'local':
+            if not blocking:
+                raise BioLibError('The argument "blocking" cannot be False when running locally')
 
-        BiolibJobApi.update_state(job['public_id'], JobState.IN_PROGRESS.value)
+            if experiment_id:
+                logger.warning('The argument "experiment_id" is ignored when running locally')
 
-        try:
-            module_output = run_job(
-                job,
-                module_input_serialized,
-                force_local=(machine == 'local'),
-                result_name_prefix=result_prefix,
+            if result_prefix:
+                logger.warning('The argument "result_prefix" is ignored when running locally')
+
+            return self._run_locally(module_input_serialized)
+
+        job = self._start_in_cloud(
+            experiment_id=experiment_id,
+            machine=machine,
+            module_input_serialized=module_input_serialized,
+            override_command=override_command,
+            result_prefix=result_prefix,
+        )
+        if blocking:
+            # TODO: Deprecate utils.STREAM_STDOUT and always stream logs by simply calling job.stream_logs()
+            if utils.IS_RUNNING_IN_NOTEBOOK:
+                utils.STREAM_STDOUT = True
+
+            enable_print = bool(
+                utils.STREAM_STDOUT and
+                (self._app_version.get('main_output_file') or self._app_version.get('stdout_render_type') == 'text')
             )
-            try:
-                BiolibJobApi.update_state(job_id=job['public_id'], state=JobState.COMPLETED.value)
-            except Exception as error:  # pylint: disable=broad-except
-                logger.debug(f'Could not update job state to completed:\n{error}')
-
-            # TODO: Create new JobDict that has shared relevant fields
-            job_dict = cast(JobDict, job)
-            job_object = Job(job_dict=job_dict)
-            job_object._set_result_module_output(module_output)  # pylint: disable=protected-access
-            return job_object
-
-        except BioLibError as exception:
-            logger.error(f'Compute failed with: {exception.message}')
-            try:
-                BiolibApiClient.refresh_auth_token()
-                BiolibJobApi.update_state(job_id=job['public_id'], state=JobState.FAILED.value)
-            except Exception as error:  # pylint: disable=broad-except
-                logger.debug(f'Could not update job state to failed:\n{error}')
-
-            raise exception
+            job._stream_logs(enable_print=enable_print)  # pylint: disable=protected-access
+
+        return job
 
     def exec(self, args=None, stdin=None, files=None, machine=''):
         return self.cli(args, stdin, files, override_command=True, machine=machine)
@@ -146,7 +131,9 @@ Example: "app.cli('--help')"
                 if os.path.isfile(arg) or os.path.isdir(arg):
                     files.append(arg)
                     args[idx] = arg.rstrip('/').split('/')[-1]
-                elif os.path.isfile(arg.split("=")[-1]) or os.path.isdir(arg.split("=")[-1]): # support --myarg=file.txt
+
+                # support --myarg=file.txt
+                elif os.path.isfile(arg.split("=")[-1]) or os.path.isdir(arg.split("=")[-1]):
                     files.append(arg.split("=")[-1])
                     args[idx] = arg.split("=")[0] + '=' + arg.split("=")[-1].rstrip('/').split('/')[-1]
                 else:
@@ -202,18 +189,15 @@ Example: "app.cli('--help')"
         )
         return module_input_serialized
 
-    def _cli_non_blocking(
+    def _start_in_cloud(
             self,
-            args: Optional[List[str]] = None,
-            stdin: Optional[bytes] = None,
-            files: Optional[List[str]] = None,
+            module_input_serialized: bytes,
             override_command: bool = False,
             machine: Optional[str] = None,
             experiment_id: Optional[str] = None,
             result_prefix: Optional[str] = None,
     ) -> Job:
-        module_input_serialized = self._get_serialized_module_input(args, stdin, files)
-        if len(module_input_serialized) < 10_000_000:
+        if len(module_input_serialized) < 500_000:
             _job_dict = BiolibJobApi.create_job_with_data(
                 app_version_uuid=self._app_version['public_id'],
                 arguments_override_command=override_command,
@@ -235,11 +219,23 @@ Example: "app.cli('--help')"
         logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
         return Job(cast(JobDict, job_dict))
 
+    def _run_locally(self, module_input_serialized: bytes) -> Job:
+        job_dict = BiolibJobApi.create(app_version_id=self._app_version['public_id'])
+        job = Job(job_dict)
+        try:
+            BiolibJobApi.update_state(job.id, JobState.IN_PROGRESS)
+            module_output = JobWorker().run_job_locally(job_dict, module_input_serialized)
+            job._set_result_module_output(module_output)  # pylint: disable=protected-access
+            BiolibJobApi.update_state(job.id, JobState.COMPLETED)
+        except BaseException:
+            BiolibJobApi.update_state(job.id, JobState.FAILED)
+        return job
+
     def run(self, **kwargs) -> Job:
         args = []
         for key, value in kwargs.items():
             if isinstance(value, dict):
-                value =  io.StringIO(json.dumps(value))
+                value = io.StringIO(json.dumps(value))
 
             if not key.startswith('--'):
                 key = f'--{key}'

biolib/biolib_api_client/biolib_job_api.py

@@ -8,9 +8,10 @@ import biolib.api
 
 from biolib import utils
 from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient, CloudJob
+from biolib.biolib_api_client import BiolibApiClient, CloudJob, JobState
 from biolib.biolib_errors import BioLibError, RetryLimitException, StorageDownloadFailed, JobResultPermissionError, \
     JobResultError, JobResultNotFound
+from biolib.biolib_logging import logger
 from biolib.utils import BIOLIB_PACKAGE_VERSION
 from biolib.typing_utils import TypedDict, Optional, Literal, Dict
 
@@ -75,18 +76,11 @@ class BiolibJobApi:
         return response.json()
 
     @staticmethod
-    def update_state(job_id, state):
-        response = requests.patch(
-            f'{BiolibApiClient.get().base_url}/api/jobs/{job_id}/',
-            json={'state': state},
-            auth=BearerAuth(BiolibApiClient.get().access_token)
-        )
-
-        # TODO: Error handling with response object
-        if not response.ok:
-            raise BioLibError(response.content)
-
-        return response.json()
+    def update_state(job_uuid: str, state: JobState) -> None:
+        try:
+            biolib.api.client.patch(path=f'/jobs/{job_uuid}/', data={'state': state.value})
+        except BaseException as error:
+            logger.error(f'Failed to update job "{job_uuid}" to state "{state.value}" due to {error}')
 
     @staticmethod
     def create_cloud_job(job_id: str, result_name_prefix: Optional[str]) -> CloudJob:

biolib/compute_node/job_worker/executors/__init__.py

@@ -1,4 +1,2 @@
 from biolib.compute_node.job_worker.executors.docker_executor import DockerExecutor
-from biolib.compute_node.job_worker.executors.remote import RemoteExecutor
-
 from biolib.compute_node.job_worker.executors.types import *

biolib/compute_node/job_worker/job_worker.py

@@ -3,6 +3,7 @@ import json
 import socket
 import shlex
 import sys
+import tempfile
 import zipfile
 from time import time
 from queue import Queue
@@ -36,7 +37,8 @@ from biolib.compute_node.socket_sender_thread import SocketSenderThread
 from biolib.compute_node.job_worker.mappings import Mappings, path_without_first_folder
 from biolib.compute_node.job_worker.utils import ComputeProcessException, log_disk_and_memory_usage_info
 from biolib.compute_node.utils import get_package_type, SystemExceptionCodes, SystemExceptionCodeMap
-from biolib.biolib_binary_format import SavedJob, SystemStatusUpdate, ModuleInput, SystemException
+from biolib.biolib_binary_format import SavedJob, SystemStatusUpdate, ModuleInput, SystemException, \
+    UnencryptedModuleOutput, InMemoryIndexableBuffer
 
 SOCKET_HOST = '127.0.0.1'
 
@@ -599,18 +601,22 @@ class JobWorker:
         self._send_status_update(StatusUpdate(progress=94, log_message='Computation finished'))
         return main_module_output_path
 
-    def run_job_locally(self, job: CreatedJobDict, module_input_serialized: bytes, job_temporary_dir: str) -> str:
+    def run_job_locally(self, job_dict: CreatedJobDict, module_input_serialized: bytes) -> UnencryptedModuleOutput:
         try:
-            self.job_temporary_dir = job_temporary_dir
-            self._root_job_wrapper = JobWrapper(
-                access_token=BiolibApiClient.get().access_token or '',
-                BASE_URL=BiolibApiClient.get().base_url,
-                cloud_job=None,
-                compute_node_info=None,
-                job=job,
-                job_temporary_dir=job_temporary_dir,
-            )
-            self._start_network_and_remote_host_proxies(job)
-            return self._run_root_job(module_input_serialized)
+            with tempfile.TemporaryDirectory() as job_temporary_dir:
+                self.job_temporary_dir = job_temporary_dir
+                self._root_job_wrapper = JobWrapper(
+                    access_token=BiolibApiClient.get().access_token or '',
+                    BASE_URL=BiolibApiClient.get().base_url,
+                    cloud_job=None,
+                    compute_node_info=None,
+                    job=job_dict,
+                    job_temporary_dir=job_temporary_dir,
+                )
+                self._start_network_and_remote_host_proxies(job_dict)
+                module_output_path = self._run_root_job(module_input_serialized)
+                with open(module_output_path, mode='rb') as module_output_file:
+                    module_output_serialized = module_output_file.read()
+                return UnencryptedModuleOutput(InMemoryIndexableBuffer(module_output_serialized))
         finally:
             self._cleanup()

{pybiolib-1.1.1151.dist-info → pybiolib-1.1.1172.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1151
+Version: 1.1.1172
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.1151.dist-info → pybiolib-1.1.1172.dist-info}/RECORD

@@ -4,16 +4,15 @@ biolib/__init__.py,sha256=yoQA2fWt8aToFas3sx6UIoFBW7-dd2yf7Nyi5smQl88,3418
 biolib/api/__init__.py,sha256=fhsbqVxDYfQ2NMYKkFW-msplAa1YPllfKEkAtLdD9IM,95
 biolib/api/client.py,sha256=uQhoqFjUS11Fmdm2TB8qTMAAVnvULq1uTWN3cEZKOQc,4954
 biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
-biolib/app/app.py,sha256=IFf8qoAKdhI55WcmBtw4s8GRLtwte6KjM9bWqIEkFI0,9193
+biolib/app/app.py,sha256=2_srxC4zHzANUjwdRWMAPYV4H5yww1oGI-e6pscHZJM,9083
 biolib/app/search_apps.py,sha256=22rJ_vIxA5GKC6wdDIAjMCe5eOPoPD4BYZuyQ0OddW8,1493
-biolib/app/utils.py,sha256=cvIROstlqJJBtYmnQ1DpkD2j0QmpTnoNqQV5G3UXXWQ,1515
 biolib/biolib_api_client/__init__.py,sha256=E5EMa19wJoblwSdQPYrxc_BtIeRsAuO0L_jQweWw-Yk,182
 biolib/biolib_api_client/api_client.py,sha256=ny1J8CXY9AO99PRxoI36fAIxNPMIwAMX_69u4vIec1g,5574
 biolib/biolib_api_client/app_types.py,sha256=mLQ6h8w2Zw1bVCq3ZVEIsCkLi8cNmH5_OoVRNpZp4co,2397
 biolib/biolib_api_client/auth.py,sha256=mmfeH1ujDRH3_RodqagYtPAfCbFe4SYtJGuFeJzK-UQ,1668
 biolib/biolib_api_client/biolib_account_api.py,sha256=2Mc6SbmjBSsz8lF2H3iiaZHEm47LyI0B4rjmvzxKHt4,580
 biolib/biolib_api_client/biolib_app_api.py,sha256=dxQLKgjSvaWsqPEOzt5EXd4gos0SGC1vFkzNbtiU8Lc,2859
-biolib/biolib_api_client/biolib_job_api.py,sha256=KxlZpTWLEYMVXXHePnLDiGK3swbKkOz7_tLuk-h2gnU,7142
+biolib/biolib_api_client/biolib_job_api.py,sha256=E9ctexfcg8efKx86LgAv3qQd-Guvdnj78_--PLZ96ls,7092
 biolib/biolib_api_client/biolib_large_file_system_api.py,sha256=5q4UlRw-OB2bRu7CeB14fy-azz2YeSgem5DbvMSaS44,1777
 biolib/biolib_api_client/common_types.py,sha256=RH-1KNHqUF-EkTpfPOSTt5Mq1GPdfju_cqXDesscO1I,123
 biolib/biolib_api_client/job_types.py,sha256=iAVWY7wHnJAPbbFf8_f0LzfYHb2jTaqCH-eaQ-WeXOc,1256
@@ -50,17 +49,15 @@ biolib/compute_node/job_worker/__init__.py,sha256=ipdPWaABKYrltxny15e2kK8PWdEE7V
 biolib/compute_node/job_worker/cache_state.py,sha256=N2AxvvgfizKx9P_ysM0GDCmueMUlgcveA4xSdz_Z2IQ,4822
 biolib/compute_node/job_worker/cache_types.py,sha256=4EIF2bohizsuaF61F0ufedBn6zI4SkvclRIFIxhxZuE,891
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=k5WJe1BR-jA_q5oIqXfVF2Q427X2SPzhse6U5FTlR4w,7562
-biolib/compute_node/job_worker/executors/__init__.py,sha256=8gc5uJakp08WA7mt_wY9fpzq1YPJJv_NsNQCk96ht04,221
+biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
 biolib/compute_node/job_worker/executors/docker_executor.py,sha256=iMf1EGfGCizajLTYQbBIcb9Up4VPwWMzjHD6OtbBo8I,24216
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
-biolib/compute_node/job_worker/executors/remote/__init__.py,sha256=Yjav4ARUUb3jscwJHF_XPDL0YQJR483OpDe-8i8Kl4w,91
-biolib/compute_node/job_worker/executors/remote/remote_executor.py,sha256=X-Lz9-0jWcYJX5vqht4_5Ru1dEnK2Xt1qMG9vT2rYZs,1595
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
 biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
 biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
 biolib/compute_node/job_worker/job_storage.py,sha256=cW0X9qmvpwJNFPLdHSUKrLfl0FEvh1PiikJw7tDz04Q,4846
-biolib/compute_node/job_worker/job_worker.py,sha256=PxdPPxUn1TUBWbXE-fWHZHG3A83CsbbMdxWgNlTFYhM,27740
+biolib/compute_node/job_worker/job_worker.py,sha256=I4fSyt5Hk3dz-Or-bAJpAEMdiVwinTX0AuQfawL96dw,28201
 biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO33vWtG60xg,1282
@@ -97,8 +94,8 @@ biolib/utils/cache_state.py,sha256=BFrZlV4XZIueIFzAFiPidX4hmwADKY5Y5ZuqlerF5l0,3
 biolib/utils/multipart_uploader.py,sha256=XFEz-4LRNVFNR6D_FKCecjxosY0bgaR5DGigKAyY5iI,9717
 biolib/utils/seq_util.py,sha256=CEFtNVBcnmVq_801T_jT4g1O30_iI4P-sRZ2GCU9EqU,1757
 biolib/utils/zip/remote_zip.py,sha256=NCdUnVbGCv7SfXCI-yVU-is_OnyWmLAnVpIdSvo-W4k,23500
-pybiolib-1.1.1151.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.1151.dist-info/METADATA,sha256=PtOtGUsm46tGnmK3bfv9YwJIirWKQ02HRL8bJj293sg,1574
-pybiolib-1.1.1151.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-pybiolib-1.1.1151.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.1151.dist-info/RECORD,,
+pybiolib-1.1.1172.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.1172.dist-info/METADATA,sha256=Buk0oP-lZD0ruRstuKTqD4UFkoqOvUzUAa8ilgzf37Q,1574
+pybiolib-1.1.1172.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+pybiolib-1.1.1172.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.1172.dist-info/RECORD,,

biolib/app/utils.py

@@ -1,41 +0,0 @@
-import tempfile
-
-from biolib import utils
-from biolib.biolib_binary_format import UnencryptedModuleOutput, InMemoryIndexableBuffer
-from biolib.biolib_errors import BioLibError
-from biolib.biolib_logging import logger
-from biolib.compute_node.job_worker.executors import RemoteExecutor, RemoteExecuteOptions
-from biolib.compute_node.job_worker.job_worker import JobWorker
-from biolib.typing_utils import Optional
-
-
-def run_job(
-        job,
-        module_input_serialized: bytes,
-        force_local: bool = False,
-        result_name_prefix: Optional[str] = None,
-) -> UnencryptedModuleOutput:
-    job_id = job['public_id']
-    logger.info(f'Job "{job_id}" is starting...')
-
-    if not force_local:
-        return RemoteExecutor.execute_job(
-            RemoteExecuteOptions(
-                biolib_base_url=utils.BIOLIB_BASE_URL,
-                job=job,
-                result_name_prefix=result_name_prefix,
-                root_job_id=job_id,
-            ),
-            module_input_serialized,
-        )
-
-    if not job['app_version'].get('modules'):
-        BioLibError('Unable to run the application locally')
-
-    with tempfile.TemporaryDirectory() as job_temporary_dir:
-        module_output_path = JobWorker().run_job_locally(job, module_input_serialized, job_temporary_dir)
-
-        with open(module_output_path, mode='rb') as module_output_file:
-            module_output_serialized = module_output_file.read()
-
-        return UnencryptedModuleOutput(InMemoryIndexableBuffer(module_output_serialized))

biolib/compute_node/job_worker/executors/remote/__init__.py

@@ -1 +0,0 @@
-from biolib.compute_node.job_worker.executors.remote.remote_executor import RemoteExecutor

biolib/compute_node/job_worker/executors/remote/remote_executor.py

@@ -1,39 +0,0 @@
-from biolib import utils
-from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
-from biolib.biolib_binary_format import UnencryptedModuleOutput
-from biolib.biolib_logging import logger
-from biolib.compute_node.job_worker.executors.types import RemoteExecuteOptions
-
-from biolib.compute_node.job_worker.job_storage import JobStorage
-from biolib.jobs import Job
-
-
-class RemoteExecutor:
-
-    @staticmethod
-    def execute_job(options: RemoteExecuteOptions, module_input_serialized: bytes) -> UnencryptedModuleOutput:
-        job_id = options['job']['public_id']
-        JobStorage.upload_module_input(
-            job=options['job'],
-            module_input_serialized=module_input_serialized
-        )
-        cloud_job = BiolibJobApi.create_cloud_job(job_id=job_id, result_name_prefix=options['result_name_prefix'])
-        logger.debug(f"Cloud: Job created with id {cloud_job['public_id']}")
-
-        if utils.IS_RUNNING_IN_NOTEBOOK:
-            utils.STREAM_STDOUT = True
-
-        app_version = options['job']['app_version']
-        enable_print = bool(
-            utils.STREAM_STDOUT and
-            (app_version.get('main_output_file') or app_version.get('stdout_render_type') == 'text')
-        )
-        job = Job.create_from_uuid(
-            uuid=options['job']['public_id'],
-            auth_token=options['job']['auth_token']
-        )
-        job._stream_logs(enable_print)  # pylint: disable=protected-access
-
-        BiolibApiClient.refresh_auth_token()
-        return JobStorage.get_module_output(job=options['job'])