pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/compute_node/job_worker/executors/docker_types.py

@@ -1,7 +1,15 @@
-from biolib.typing_utils import TypedDict, Any
+import enum
+
+from biolib.typing_utils import Any, TypedDict
 
 
 class Proxy(TypedDict):
     container: Any
     hostname: str
     ip: str
+
+
+class DockerDiffKind(enum.Enum):
+    CHANGED = 0
+    ADDED = 1
+    DELETED = 2

biolib/compute_node/job_worker/executors/types.py

@@ -1,12 +1,11 @@
-from docker.models.networks import Network  # type: ignore
+from docker.models.networks import Network
 
+from biolib.biolib_api_client.app_types import Module
+from biolib.biolib_api_client.job_types import CloudJob, CreatedJobDict
 from biolib.compute_node.job_worker.large_file_system import LargeFileSystem
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Callable, Optional, List, Dict
 from biolib.compute_node.remote_host_proxy import RemoteHostProxy
-from biolib.biolib_api_client.app_types import Module
-from biolib.biolib_api_client.job_types import Job
-from biolib.compute_node.utils import SystemExceptionCodes
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Callable, Dict, List, Optional, TypedDict
 
 
 class StatusUpdate(TypedDict):
@@ -16,12 +15,14 @@ class StatusUpdate(TypedDict):
 
 class RemoteExecuteOptions(TypedDict):
     biolib_base_url: str
-    job: Job
+    job: CreatedJobDict
+    result_name_prefix: Optional[str]
     root_job_id: str
 
 
 SendStatusUpdateType = Callable[[StatusUpdate], None]
-SendSystemExceptionType = Callable[[SystemExceptionCodes], None]
+SendSystemExceptionType = Callable[[int], None]
+SendStdoutAndStderrType = Callable[[bytes], None]
 
 
 class LocalExecutorOptions(TypedDict):
@@ -29,11 +30,20 @@ class LocalExecutorOptions(TypedDict):
     biolib_base_url: str
     compute_node_info: Optional[ComputeNodeInfo]
     internal_network: Optional[Network]
-    job: Job
+    job: CreatedJobDict
+    cloud_job: Optional[CloudJob]
     large_file_systems: Dict[str, LargeFileSystem]
     module: Module
+    module_input_path: str
+    module_output_path: str
     remote_host_proxies: List[RemoteHostProxy]
     root_job_id: str
     runtime_zip_bytes: Optional[bytes]  # TODO: replace this with a module_source_serialized
     send_status_update: SendStatusUpdateType
     send_system_exception: SendSystemExceptionType
+    send_stdout_and_stderr: SendStdoutAndStderrType
+
+
+class MetadataToSaveOutput(TypedDict):
+    arguments: List[str]
+    startup_error_string: Optional[str]

biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py

@@ -0,0 +1,30 @@
+import threading
+import time
+
+from biolib.typing_utils import Callable
+from biolib.compute_node.job_worker.utils import ComputeProcessException
+from biolib.compute_node.utils import SystemExceptionCodes
+
+
+class JobLegacyInputWaitTimeout(threading.Thread):
+
+    def __init__(self, job_uuid: str, input_max_wait_in_seconds: int, send_system_exception: Callable[[int], None]):
+        threading.Thread.__init__(self, daemon=True)
+        self._job_uuid = job_uuid
+        self._input_max_wait_in_seconds = input_max_wait_in_seconds
+        self._send_system_exception = send_system_exception
+        self._stop_event = threading.Event()
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    def run(self) -> None:
+        time.sleep(self._input_max_wait_in_seconds)
+
+        if not self._stop_event.is_set():
+            raise ComputeProcessException(
+                biolib_error_code=SystemExceptionCodes.FAILED_TO_GET_REQUIRED_DATA_FOR_COMPUTE.value,
+                may_contain_user_data=False,
+                original_error=Exception(f'Job "{self._job_uuid}" exceeded max wait time for legacy input.'),
+                send_system_exception=self._send_system_exception,
+            )

biolib/compute_node/job_worker/job_max_runtime_timer_thread.py

@@ -3,21 +3,19 @@ import threading
 import time
 
 from biolib.biolib_logging import logger
-from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker.utils import ComputeProcessException
 from biolib.compute_node.utils import SystemExceptionCodes
 
 
 class JobMaxRuntimeTimerThread(threading.Thread):
 
-    def __init__(self,  job_worker):
+    def __init__(self,  job_worker, max_runtime_in_seconds):
         threading.Thread.__init__(self, daemon=True)
         self._job_worker = job_worker
+        self._max_runtime_in_seconds = max_runtime_in_seconds
 
     def run(self) -> None:
-        config = CloudUtils.get_webserver_config()
-        seconds_to_wait = config['shutdown_times']['job_max_runtime_shutdown_time_in_seconds']
-        time.sleep(seconds_to_wait)
+        time.sleep(self._max_runtime_in_seconds)
         # Only raise exception and trigger clean up if the job has not already been cleaned up
         if not self._job_worker.is_cleaning_up:
             logger.debug("Job exceeded max run time. Raising exception")

biolib/compute_node/job_worker/job_storage.py

@@ -0,0 +1,108 @@
+import os
+
+from biolib import utils
+from biolib._internal.http_client import HttpClient
+from biolib.biolib_api_client import CreatedJobDict
+from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
+from biolib.compute_node.cloud_utils import CloudUtils
+from biolib.biolib_logging import logger_no_user_data
+from biolib.utils.multipart_uploader import get_chunk_iterator_from_file_object
+
+
+class JobStorage:
+    module_input_file_name = 'input-output.bbf'
+    module_output_file_name = 'module-output.bbf'
+
+    @staticmethod
+    def upload_module_input(job: CreatedJobDict, module_input_serialized: bytes) -> None:
+        job_uuid = job['public_id']
+        headers = {'Job-Auth-Token': job['auth_token']}
+
+        multipart_uploader = utils.MultiPartUploader(
+            start_multipart_upload_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/input/start_upload/',
+                headers=headers
+            ),
+            get_presigned_upload_url_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/input/presigned_upload_url/',
+                headers=headers
+            ),
+            complete_upload_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/input/complete_upload/',
+                headers=headers
+            ),
+        )
+
+        multipart_uploader.upload(
+            payload_iterator=utils.get_chunk_iterator_from_bytes(module_input_serialized),
+            payload_size_in_bytes=len(module_input_serialized),
+        )
+
+    @staticmethod
+    def upload_module_output(job_uuid: str, job_temporary_dir: str) -> None:
+        logger_no_user_data.debug(f'Job "{job_uuid}" uploading result...')
+        module_output_path = os.path.join(job_temporary_dir, JobStorage.module_output_file_name)
+        module_output_size = os.path.getsize(module_output_path)
+
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        max_chunk_count = 9_000
+        min_chunk_size_bytes = 50_000_000
+        chunk_size_in_bytes = max(min_chunk_size_bytes, module_output_size // max_chunk_count)
+
+        logger_no_user_data.debug(
+            f'Job "{job_uuid}" uploading result of size {module_output_size} bytes '
+            f'with chunk size of {chunk_size_in_bytes} bytes...'
+        )
+
+        with open(module_output_path, mode='rb') as module_output_file:
+            module_output_iterator = get_chunk_iterator_from_file_object(
+                file_object=module_output_file,
+                chunk_size_in_bytes=chunk_size_in_bytes,
+            )
+            multipart_uploader = JobStorage._get_module_output_uploader(job_uuid)
+            multipart_uploader.upload(
+                payload_iterator=module_output_iterator,
+                payload_size_in_bytes=module_output_size,
+            )
+
+        logger_no_user_data.debug(f'Job "{job_uuid}" result uploaded successfully')
+
+    @staticmethod
+    def _get_module_output_uploader(job_uuid: str) -> utils.MultiPartUploader:
+        config = CloudUtils.get_webserver_config()
+        compute_node_auth_token = config['compute_node_info']['auth_token']  # pylint: disable=unsubscriptable-object
+        headers = {'Compute-Node-Auth-Token': compute_node_auth_token}
+
+        return utils.MultiPartUploader(
+            start_multipart_upload_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/results/start_upload/',
+                headers=headers,
+            ),
+            get_presigned_upload_url_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/results/presigned_upload_url/',
+                headers=headers,
+            ),
+            complete_upload_request=dict(
+                requires_biolib_auth=False,
+                path=f'/jobs/{job_uuid}/storage/results/complete_upload/',
+                headers=headers,
+            ),
+        )
+
+    @staticmethod
+    def download_module_input(job: CreatedJobDict, path: str):
+        job_uuid = job['public_id']
+        logger_no_user_data.debug(f'Job "{job_uuid}" getting module input url...')
+        presigned_download_url = BiolibJobApi.get_job_storage_download_url(
+            job_uuid=job_uuid,
+            job_auth_token=job['auth_token'],
+            storage_type='input',
+        )
+        logger_no_user_data.debug(f'Job "{job_uuid}" downloading module input...')
+        HttpClient.request(url=presigned_download_url, response_path=path)
+        logger_no_user_data.debug(f'Job "{job_uuid}" module input downloaded')