pybiolib 1.2.339__py3-none-any.whl → 1.2.340.dev1__py3-none-any.whl

biolib/_internal/utils/multinode.py

@@ -84,6 +84,8 @@ def fasta_batch_records(records, work_per_batch_min, work_per_residue=1, verbose
             n_seqs, n_res = batch_dict['records'], batch_dict['residues']
             print(f'Batch {i+1}: {n_res} residues from {n_seqs} sequences')
 
+    # records = SeqUtil.parse_fasta(fasta_file)
+
     batches = []
     batch = []
     current_work_units = 0

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -27,7 +27,7 @@ from biolib.compute_node import utils as compute_node_utils
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker.docker_image_cache import DockerImageCache
 from biolib.compute_node.job_worker.executors.docker_types import DockerDiffKind
-from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions, StatusUpdate
+from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions, StatusUpdate, MetadataToSaveOutput
 from biolib.compute_node.job_worker.mappings import Mappings, path_without_first_folder
 from biolib.compute_node.job_worker.utilization_reporter_thread import UtilizationReporterThread
 from biolib.compute_node.job_worker.utils import ComputeProcessException
@@ -61,9 +61,12 @@ class DockerExecutor:
         os.makedirs(user_data_tar_dir, exist_ok=True)
 
         self._docker_container: Optional[Container] = None
+        self._docker_api_client = BiolibDockerClient.get_docker_client().api
         self._runtime_tar_path = f'{user_data_tar_dir}/runtime_{self._random_docker_id}.tar'
         self._input_tar_path = f'{user_data_tar_dir}/input_{self._random_docker_id}.tar'
 
+        self._metadata_for_save_output_on_cancel: Optional[MetadataToSaveOutput] = None
+
         if utils.IS_RUNNING_IN_CLOUD and not utils.BIOLIB_SECRETS_TMPFS_PATH:
             error_message = 'Running in cloud but no TMPFS path has been set for secrets'
             logger_no_user_data.error(error_message)
@@ -81,12 +84,12 @@ class DockerExecutor:
             raise Exception('Docker container was None')
         return self._docker_container
 
-    def execute_module(self, module_input_path: str, module_output_path: str) -> None:
+    def execute_module(self) -> None:
         try:
             job_uuid = self._options['job']['public_id']
             send_status_update = self._options['send_status_update']
             logger_no_user_data.debug(f'Reading module input of {job_uuid}.')
-            with open(module_input_path, 'rb') as fp:
+            with open(self._options['module_input_path'], 'rb') as fp:
                 module_input_tmp = ModuleInput(fp.read())
                 logger_no_user_data.debug(f'Deserialing module input of {job_uuid}...')
                 module_input = module_input_tmp.deserialize()
@@ -100,7 +103,7 @@ class DockerExecutor:
 
             logger_no_user_data.debug(f'Starting execution of {job_uuid}.')
             try:
-                self._execute_helper(module_input, module_output_path)
+                self._execute_helper(module_input)
             except docker.errors.NotFound as docker_error:
                 raise DockerContainerNotFoundDuringExecutionException from docker_error
             except Exception as exception:
@@ -161,7 +164,8 @@ class DockerExecutor:
             may_contain_user_data=False,
         )
 
-    def _execute_helper(self, module_input, module_output_path: str) -> None:
+
+    def _execute_helper(self, module_input) -> None:
         job_uuid = self._options['job']['public_id']
         logger_no_user_data.debug(f'Initializing container for {job_uuid}.')
         self._initialize_docker_container(module_input)
@@ -187,10 +191,9 @@ class DockerExecutor:
         logger_no_user_data.debug(f'_map_and_copy_input_files_to_container for {job_uuid}.')
         self._map_and_copy_input_files_to_container(module_input['files'], module_input['arguments'])
 
-        docker_api_client = BiolibDockerClient.get_docker_client().api
         logger_no_user_data.debug(f'Attaching Docker container for {job_uuid}')
 
-        stdout_and_stderr_stream = docker_api_client.attach(
+        stdout_and_stderr_stream = self._docker_api_client.attach(
             container=self._container.id,
             stderr=True,
             stdout=True,
@@ -211,6 +214,12 @@ class DockerExecutor:
             logger.debug(f'Warning: Job "{job_uuid}" failed to start container. Hit error: {startup_error_string}')
             # even though the container start failed we should still be able to call logs() and wait() on it, so we pass
 
+        self._metadata_for_save_output_on_cancel = MetadataToSaveOutput(
+            arguments=module_input['arguments'],
+            startup_error_string=startup_error_string,
+            pre_start_diff=pre_start_diff,
+        )
+
         if self._options['job']['app_version'].get('stdout_render_type') != 'markdown':
             logger_no_user_data.debug(f'Streaming stdout for {job_uuid}')
             for stdout_and_stderr in stdout_and_stderr_stream:
@@ -220,7 +229,14 @@ class DockerExecutor:
                 self._send_stdout_and_stderr(stdout_and_stderr)
 
         logger_no_user_data.debug(f'Waiting on docker for {job_uuid}')
-        docker_result = docker_api_client.wait(self._container.id)
+        try:
+            docker_result = self._docker_api_client.wait(self._container.id)
+        except docker.errors.NotFound as error:
+            if self._is_cleaning_up:
+                return
+            else:
+                raise error
+
         logger_no_user_data.debug(f'Got result from docker for {job_uuid}')
         exit_code = docker_result['StatusCode']
         # 137 is the error code from linux OOM killer (Should catch 90% of OOM errors)
@@ -232,22 +248,26 @@ class DockerExecutor:
             )
 
         logger_no_user_data.debug(f'Docker container exited with code {exit_code} for {job_uuid}')
+        self._save_module_output_from_container(exit_code, self._metadata_for_save_output_on_cancel)
+
 
-        full_stdout = docker_api_client.logs(self._container.id, stdout=True, stderr=False)
-        full_stderr = docker_api_client.logs(self._container.id, stdout=False, stderr=True)
+    def _save_module_output_from_container(self, exit_code: int, metadata: MetadataToSaveOutput) -> None:
+        full_stdout = self._docker_api_client.logs(self._container.id, stdout=True, stderr=False)
+        full_stderr = self._docker_api_client.logs(self._container.id, stdout=False, stderr=True)
 
-        if startup_error_string:
-            full_stderr = full_stderr + startup_error_string.encode()
+        if metadata['startup_error_string']:
+            full_stderr = full_stderr + metadata['startup_error_string'].encode()
 
         self._write_module_output_to_file(
-            arguments=module_input['arguments'],
+            arguments=metadata['arguments'],
             exit_code=exit_code,
-            module_output_path=module_output_path,
+            module_output_path=self._options['module_output_path'],
             stderr=full_stderr,
             stdout=full_stdout,
-            pre_start_diff=pre_start_diff,
+            pre_start_diff=metadata['pre_start_diff'],
         )
 
+
     def cleanup(self):
         # Don't clean up if already in the process of doing so, or done doing so
         if self._is_cleaning_up:
@@ -255,6 +275,22 @@ class DockerExecutor:
         else:
             self._is_cleaning_up = True
 
+        if self._metadata_for_save_output_on_cancel is not None:
+            try:
+                logger_no_user_data.debug('Attempting to save results')
+                self._docker_container.stop()
+                self._docker_container.reload()
+                logger_no_user_data.debug(f'Container state {self._docker_container.status}')
+                self._save_module_output_from_container(
+                    exit_code=self._docker_container.attrs['State']['ExitCode'],
+                    metadata=self._metadata_for_save_output_on_cancel
+                )
+                logger_no_user_data.debug('Saved results')
+            except BaseException as error:
+                logger_no_user_data.error(f'Failed to save results on cancel with error: {error}')
+        else:
+            logger_no_user_data.debug('Missing metadata, cannot save results')
+
         tar_time = time.time()
         for path_to_delete in [self._input_tar_path, self._runtime_tar_path]:
             if os.path.exists(path_to_delete):

biolib/compute_node/job_worker/executors/types.py

@@ -34,9 +34,16 @@ class LocalExecutorOptions(TypedDict):
     cloud_job: Optional[CloudJob]
     large_file_systems: Dict[str, LargeFileSystem]
     module: Module
+    module_input_path: str
+    module_output_path: str
     remote_host_proxies: List[RemoteHostProxy]
     root_job_id: str
     runtime_zip_bytes: Optional[bytes]  # TODO: replace this with a module_source_serialized
     send_status_update: SendStatusUpdateType
     send_system_exception: SendSystemExceptionType
     send_stdout_and_stderr: SendStdoutAndStderrType
+
+class MetadataToSaveOutput(TypedDict):
+    arguments: List[str]
+    startup_error_string: Optional[str]
+    pre_start_diff: List[Dict]

biolib/compute_node/job_worker/job_worker.py

@@ -400,15 +400,15 @@ class JobWorker:
                 cloud_job=self._root_job_wrapper['cloud_job'],
                 large_file_systems=lfs_dict,
                 module=main_module,
+                module_input_path=module_input_path,
+                module_output_path=main_module_output_path,
                 remote_host_proxies=self._remote_host_proxies,
                 root_job_id=root_job_id,
                 runtime_zip_bytes=runtime_zip_bytes,
                 send_status_update=self._send_status_update,
                 send_system_exception=self.send_system_exception,
                 send_stdout_and_stderr=self.send_stdout_and_stderr,
-            ),
-            module_input_path,
-            main_module_output_path,
+            )
         )
 
         if utils.IS_RUNNING_IN_CLOUD:
@@ -418,11 +418,11 @@ class JobWorker:
     def _run_module(
             self,
             options: LocalExecutorOptions,
-            module_input_path: str,
-            module_output_path: str,
     ) -> None:
         module = options['module']
         job_id = options['job']['public_id']
+        module_output_path = options['module_output_path']
+        module_input_path = options['module_input_path']
         logger_no_user_data.debug(f'Job "{job_id}" running module "{module["name"]}"...')
 
         executor_instance: DockerExecutor
@@ -469,7 +469,7 @@ class JobWorker:
             # Log memory and disk before pulling and executing module
             log_disk_and_memory_usage_info()
 
-        executor_instance.execute_module(module_input_path, module_output_path)
+        executor_instance.execute_module()
 
     def _connect_to_parent(self):
         try:

biolib/compute_node/webserver/worker_thread.py

@@ -7,6 +7,7 @@ import sys
 import threading
 import time
 from queue import Queue
+from typing import Optional
 
 from biolib import api, utils
 from biolib.biolib_binary_format import ModuleOutputV2, SystemException, SystemStatusUpdate
@@ -55,6 +56,26 @@ class WorkerThread(threading.Thread):
     def _job_temporary_dir(self):
         return self.compute_state['job_temporary_dir']
 
+    def _upload_module_output_and_get_exit_code(self) -> Optional[int]:
+        exit_code = None
+        try:
+            module_output_path = os.path.join(
+                self._job_temporary_dir,
+                JobStorage.module_output_file_name,
+            )
+            if os.path.exists(module_output_path):
+                module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
+                exit_code = module_output.get_exit_code()
+                logger_no_user_data.debug(f'Got exit code: {exit_code}')
+                if utils.IS_RUNNING_IN_CLOUD:
+                    JobStorage.upload_module_output(
+                        job_temporary_dir=self._job_temporary_dir,
+                        job_uuid=self._job_uuid,
+                    )
+        except Exception as error:
+            logger_no_user_data.error(f'Could not upload module output or get exit code: {error}')
+        return exit_code
+
     def run(self):
         try:
             while True:
@@ -77,25 +98,7 @@ class WorkerThread(threading.Thread):
 
                     # If 'Computation Finished'
                     if progress == 94:
-                        # Get Job exit code
-                        try:
-                            module_output_path = os.path.join(
-                                self._job_temporary_dir,
-                                JobStorage.module_output_file_name,
-                            )
-                            module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
-                            self.compute_state['exit_code'] = module_output.get_exit_code()
-                            logger_no_user_data.debug(f"Got exit code: {self.compute_state['exit_code']}")
-
-                        except Exception as error:  # pylint: disable=broad-except
-                            logger_no_user_data.error(f'Could not get exit_code from module output due to: {error}')
-
-                        if utils.IS_RUNNING_IN_CLOUD:
-                            JobStorage.upload_module_output(
-                                job_temporary_dir=self._job_temporary_dir,
-                                job_uuid=self._job_uuid,
-                            )
-
+                        self.compute_state['exit_code'] = self._upload_module_output_and_get_exit_code()
                         self._set_status_update(progress=95, log_message='Result Ready')
                         self.compute_state['is_completed'] = True
                         self.terminate()
@@ -174,13 +177,13 @@ class WorkerThread(threading.Thread):
 
     def terminate(self) -> None:
         cloud_job_uuid = self.compute_state['cloud_job_id']
-        exit_code = self.compute_state.get('exit_code')
         system_exception_code = self.compute_state['status'].get('error_code')
-        if utils.IS_RUNNING_IN_CLOUD:
+
+        if utils.IS_RUNNING_IN_CLOUD and system_exception_code != SystemExceptionCodes.CANCELLED_BY_USER.value:
             CloudUtils.finish_cloud_job(
                 cloud_job_id=cloud_job_uuid,
                 system_exception_code=system_exception_code,
-                exit_code=exit_code,
+                exit_code=self.compute_state.get('exit_code', None),
             )
 
         deregistered_due_to_error = False
@@ -211,6 +214,15 @@ class WorkerThread(threading.Thread):
                     CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
+            elif system_exception_code == SystemExceptionCodes.CANCELLED_BY_USER.value:
+                self.compute_state['exit_code'] = self._upload_module_output_and_get_exit_code()
+                CloudUtils.finish_cloud_job(
+                    cloud_job_id=cloud_job_uuid,
+                    system_exception_code=system_exception_code,
+                    exit_code=self.compute_state.get('exit_code', None),
+                )
+
+
         # Delete result as error occurred
         if system_exception_code and os.path.exists(self._job_temporary_dir):
             shutil.rmtree(self._job_temporary_dir)

{pybiolib-1.2.339.dist-info → pybiolib-1.2.340.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybiolib
-Version: 1.2.339
+Version: 1.2.340.dev1
 Summary: BioLib Python Client
 License: MIT
 Keywords: biolib

{pybiolib-1.2.339.dist-info → pybiolib-1.2.340.dev1.dist-info}/RECORD

@@ -24,7 +24,7 @@ biolib/_internal/types/resource.py,sha256=G-vPkZoe4Um6FPxsQZtRzAlbSW5sDW4NFkbjn2
 biolib/_internal/types/resource_version.py,sha256=sLxViYXloDDUhTDFgjegiQCj097OM1Ih1-uqlC_4ULA,174
 biolib/_internal/types/typing.py,sha256=D4EKKEe7kDx0K6lJi-H_XLtk-8w6nu2fdqn9bvzI-Xo,288
 biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
-biolib/_internal/utils/multinode.py,sha256=-J3PEAK3NaOwCn--5T7vWHkA3yu5w9QhmuhkQcH-2wY,8229
+biolib/_internal/utils/multinode.py,sha256=G0t7dkHtoxIfdCD0vjG8eyQwRxQ0zWazOWZUFQ3hQxw,8278
 biolib/_runtime/runtime.py,sha256=bZQ0m39R9jOBVAtlyvzDnOobKueOAQUCwMUZjDQnO7E,4439
 biolib/api/__init__.py,sha256=mQ4u8FijqyLzjYMezMUUbbBGNB3iFmkNdjXnWPZ7Jlw,138
 biolib/api/client.py,sha256=_kmwaGI_-u7kOeWVCYmo-pD2K1imwEv9n2gNZRb5F-I,3790
@@ -76,14 +76,14 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=J-wv7FTb_HJnfRdIByiOdtj18Fw5nYvVm-n0D1mrqwA,29377
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=NpDSgNWbnC0Qr6aFaQMEfhib3O9BMOPsia-YqeTMhpU,30966
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
+biolib/compute_node/job_worker/executors/types.py,sha256=gw_T-oq0lty8Hg8WsTrqhtV-kyOF_l5nr9Xe8-kzetc,1657
 biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
 biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
 biolib/compute_node/job_worker/job_storage.py,sha256=lScHI3ubcHKagSEW243tgbIWXUfbWDHDjEOPMvXxJE8,4603
-biolib/compute_node/job_worker/job_worker.py,sha256=fuWoYJo9HOqLmWl8yeCXh0mhT4ebbkrWac-BVb58khs,28842
+biolib/compute_node/job_worker/job_worker.py,sha256=xp1laEWUhUsKzF4ITdsTWcQlSn133pAjkeiPMAG5jqQ,28892
 biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
@@ -97,7 +97,7 @@ biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLU
 biolib/compute_node/webserver/webserver.py,sha256=o4kOAStsqThUtKlnRE-U5TP0JIYntuySDjU7PH310xg,6620
 biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
 biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
-biolib/compute_node/webserver/worker_thread.py,sha256=GRRBUqXdMKvbjyLQhYlqGIbFKeU2iiEXIe5IXi9wgdg,11806
+biolib/compute_node/webserver/worker_thread.py,sha256=0kb7RgwDSIzTZfjttUHelyAS8g7VPx1J1E7GbqYObu0,12288
 biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/experiments/experiment.py,sha256=m7FB7hot8CMOVrNyWGPB6FTkWQJwI0ytpIvtPxfoPME,7684
 biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
@@ -118,8 +118,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=Ozk0blGtPur_D9MwShD02r_mphyQmgZkx-lOHOwnlIM,6730
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.339.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.339.dist-info/METADATA,sha256=7w5znJtkMyO9KQ_CD0PdX2Hz1pBuTjnEOeSnfaOMrsk,1570
-pybiolib-1.2.339.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-pybiolib-1.2.339.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.339.dist-info/RECORD,,
+pybiolib-1.2.340.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.340.dev1.dist-info/METADATA,sha256=nZlIvyDlGZX801BNcASFHgpASI2ztyIKhkBMgoWfhss,1575
+pybiolib-1.2.340.dev1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+pybiolib-1.2.340.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.340.dev1.dist-info/RECORD,,