pybiolib 1.1.1896__py3-none-any.whl → 1.1.1907__py3-none-any.whl

biolib/_internal/data_record/data_record.py

@@ -1,6 +1,4 @@
 import os
-import shutil
-import urllib.request
 from collections import namedtuple
 from datetime import datetime
 from fnmatch import fnmatch
@@ -56,10 +54,7 @@ class DataRecord:
         remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
             resource_version_uuid=app_response['app_version']['public_id'],
         )
-        with urllib.request.urlopen(remote_storage_endpoint.get_remote_url()) as response, open(
-            output_path, 'wb'
-        ) as out_file:
-            shutil.copyfileobj(response, out_file)
+        HttpClient.request(url=remote_storage_endpoint.get_remote_url(), response_path=output_path)
 
     def download_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
         filtered_files = self.list_files(path_filter=path_filter)

biolib/_internal/http_client.py

@@ -1,5 +1,6 @@
 import json
 import platform
+import shutil
 import socket
 import ssl
 import subprocess
@@ -41,15 +42,23 @@ class HttpError(urllib.error.HTTPError):
 
 
 class HttpResponse:
-    def __init__(self, response) -> None:
+    def __init__(self, response, response_path) -> None:
         self.headers: Dict[str, str] = dict(response.headers)
         self.status_code: int = int(response.status)
-        self.content: bytes = response.read()
+        self.response_path = response_path
+        if self.response_path:
+            shutil.copyfileobj(response, self.response_path)
+        else:
+            self.content: bytes = response.read()
         self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
-        return cast(str, self.content.decode('utf-8'))
+        if self.response_path:
+            with open(self.response_path, 'rb') as fp:
+                return cast(str, fp.read().decode('utf-8'))
+        else:
+            return cast(str, self.content.decode('utf-8'))
 
     def json(self):
         return json.loads(self.text)
@@ -66,6 +75,7 @@ class HttpClient:
         headers: Optional[Dict[str, str]] = None,
         retries: int = 5,
         timeout_in_seconds: Optional[int] = None,
+        response_path: Optional[str] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
             HttpClient.ssl_context = _create_ssl_context()
@@ -94,7 +104,7 @@ class HttpClient:
                     context=HttpClient.ssl_context,
                     timeout=timeout_in_seconds,
                 ) as response:
-                    return HttpResponse(response)
+                    return HttpResponse(response, response_path)
 
             except urllib.error.HTTPError as error:
                 if error.code == 502:

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -83,11 +83,12 @@ class DockerExecutor:
             raise Exception('Docker container was None')
         return self._docker_container
 
-    def execute_module(self, module_input_serialized: bytes, module_output_path: str) -> None:
+    def execute_module(self, module_input_path: str, module_output_path: str) -> None:
         try:
             job_uuid = self._options['job']['public_id']
             send_status_update = self._options['send_status_update']
-            module_input = ModuleInput(module_input_serialized).deserialize()
+            with open(module_input_path, 'rb') as fp:
+                module_input = ModuleInput(fp.read()).deserialize()
 
             send_status_update(StatusUpdate(progress=55, log_message='Pulling images...'))
 

biolib/compute_node/job_worker/job_storage.py

@@ -10,6 +10,7 @@ from biolib.utils.multipart_uploader import get_chunk_iterator_from_file_object
 
 
 class JobStorage:
+    module_input_file_name = 'input-output.bbf'
     module_output_file_name = 'module-output.bbf'
 
     @staticmethod
@@ -81,7 +82,7 @@ class JobStorage:
         )
 
     @staticmethod
-    def get_module_input(job: CreatedJobDict) -> bytes:
+    def download_module_input(job: CreatedJobDict, path: str):
         job_uuid = job['public_id']
         logger_no_user_data.debug(f'Job "{job_uuid}" downloading module input...')
         presigned_download_url = BiolibJobApi.get_job_storage_download_url(
@@ -89,7 +90,5 @@ class JobStorage:
             job_auth_token=job['auth_token'],
             storage_type='input',
         )
-        response = HttpClient.request(url=presigned_download_url)
-        data: bytes = response.content
+        HttpClient.request(url=presigned_download_url, response_path=path)
         logger_no_user_data.debug(f'Job "{job_uuid}" module input downloaded')
-        return data

biolib/compute_node/job_worker/job_worker.py

@@ -133,7 +133,8 @@ class JobWorker:
                         ).start()
 
                         try:
-                            module_input_serialized = JobStorage.get_module_input(job=job)
+                            module_input_path = os.path.join(self.job_temporary_dir, JobStorage.module_input_file_name)
+                            JobStorage.download_module_input(job=job, path=module_input_path)
                         except StorageDownloadFailed:
                             # Expect module input to be handled in a separate ModuleInput package
                             self._legacy_input_wait_timeout_thread = JobLegacyInputWaitTimeout(
@@ -147,7 +148,7 @@ class JobWorker:
                             raise error
 
                         try:
-                            self._run_root_job(module_input_serialized)
+                            self._run_root_job(module_input_path)
 
                         # This error occurs when trying to access the container after the job worker has cleaned it up.
                         # In that case stop the computation.
@@ -165,7 +166,9 @@ class JobWorker:
                         self._legacy_input_wait_timeout_thread.stop()
 
                     try:
-                        self._run_root_job(package)
+                        module_input_path = os.path.join(self.job_temporary_dir, JobStorage.module_input_file_name)
+                        open(module_input_path, 'wb').write(package)
+                        self._run_root_job(module_input_path)
 
                     # This error occurs when trying to access the container after the job worker has cleaned it up.
                     # In that case stop the computation.
@@ -331,15 +334,15 @@ class JobWorker:
     def _run_app_version(
             self,
             app_version_id: str,
-            module_input_serialized: bytes,
+            module_input_path: str,
             caller_job: CreatedJobDict,
             main_module_output_path: str,
     ) -> None:
         job: CreatedJobDict = BiolibJobApi.create(app_version_id, caller_job=caller_job['public_id'])
         self._jobs[job['public_id']] = job
-        self._run_job(job, module_input_serialized, main_module_output_path)
+        self._run_job(job, module_input_path, main_module_output_path)
 
-    def _run_job(self, job: CreatedJobDict, module_input_serialized: bytes, main_module_output_path: str) -> None:
+    def _run_job(self, job: CreatedJobDict, module_input_path: str, main_module_output_path: str) -> None:
         job_uuid = job['public_id']
         logger_no_user_data.info(f'Job "{job_uuid}" running...')
         if self._root_job_wrapper is None:
@@ -406,7 +409,7 @@ class JobWorker:
                 send_system_exception=self.send_system_exception,
                 send_stdout_and_stderr=self.send_stdout_and_stderr,
             ),
-            module_input_serialized,
+            module_input_path,
             main_module_output_path,
         )
 
@@ -417,15 +420,20 @@ class JobWorker:
     def _run_module(
             self,
             options: LocalExecutorOptions,
-            module_input_serialized: bytes,
+            module_input_path: str,
             module_output_path: str,
     ) -> None:
         module = options['module']
         job_id = options['job']['public_id']
         logger_no_user_data.debug(f'Job "{job_id}" running module "{module["name"]}"...')
+
         executor_instance: DockerExecutor
         if module['environment'] == ModuleEnvironment.BIOLIB_APP.value:
+            if not self.job_temporary_dir:
+                raise BioLibError('Undefined job_temporary_dir')
             logger_no_user_data.debug(f'Job "{job_id}" starting child job...')
+            with open(module_input_path,'rb') as fp:
+                module_input_serialized = fp.read()
             module_input = ModuleInput(module_input_serialized).deserialize()
             module_input_with_runtime_zip = self._add_runtime_zip_and_command_to_module_input(options, module_input)
             module_input_with_runtime_zip_serialized = ModuleInput().serialize(
@@ -433,9 +441,11 @@ class JobWorker:
                 arguments=module_input_with_runtime_zip['arguments'],
                 files=module_input_with_runtime_zip['files'],
             )
+            module_input_path_new = os.path.join(self.job_temporary_dir, "runtime." + JobStorage.module_input_file_name)
+            open(module_input_path_new, 'wb').write(module_input_with_runtime_zip_serialized)
             return self._run_app_version(
                 module['image_uri'],
-                module_input_with_runtime_zip_serialized,
+                module_input_path_new,
                 options['job'],
                 module_output_path,
             )
@@ -461,7 +471,7 @@ class JobWorker:
             # Log memory and disk before pulling and executing module
             log_disk_and_memory_usage_info()
 
-        executor_instance.execute_module(module_input_serialized, module_output_path)
+        executor_instance.execute_module(module_input_path, module_output_path)
 
     def _connect_to_parent(self):
         try:
@@ -587,7 +597,7 @@ class JobWorker:
                 may_contain_user_data=False
             ) from exception
 
-    def _run_root_job(self, module_input_serialized: bytes) -> str:
+    def _run_root_job(self, module_input_path: str) -> str:
         # Make typechecker happy
         if not self._root_job_wrapper or not self.job_temporary_dir:
             raise BioLibError('Undefined job_wrapper or job_temporary_dir')
@@ -595,7 +605,7 @@ class JobWorker:
         main_module_output_path = os.path.join(self.job_temporary_dir, JobStorage.module_output_file_name)
         self._run_job(
             job=self._root_job_wrapper['job'],
-            module_input_serialized=module_input_serialized,
+            module_input_path=module_input_path,
             main_module_output_path=main_module_output_path,
         )
         self._send_status_update(StatusUpdate(progress=94, log_message='Computation finished'))
@@ -614,7 +624,9 @@ class JobWorker:
                     job_temporary_dir=job_temporary_dir,
                 )
                 self._start_network_and_remote_host_proxies(job_dict)
-                module_output_path = self._run_root_job(module_input_serialized)
+                module_input_path = os.path.join(self.job_temporary_dir, JobStorage.module_input_file_name)
+                open(module_input_path, 'wb').write(module_input_serialized)
+                module_output_path = self._run_root_job(module_input_path)
                 with open(module_output_path, mode='rb') as module_output_file:
                     module_output_serialized = module_output_file.read()
                 return ModuleOutputV2(InMemoryIndexableBuffer(module_output_serialized))

{pybiolib-1.1.1896.dist-info → pybiolib-1.1.1907.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1896
+Version: 1.1.1907
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.1896.dist-info → pybiolib-1.1.1907.dist-info}/RECORD

@@ -3,9 +3,9 @@ README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
 biolib/__init__.py,sha256=nfZvVkrHZLvjvvlAvFzhvem9NMfqgmw8NWaCH9HGzew,4045
 biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/_internal/data_record/__init__.py,sha256=1Bk303i3rFet9veS56fIsrBYtT5X3n9vcsYMA6T6c5o,36
-biolib/_internal/data_record/data_record.py,sha256=6ap6f4RMz6FbdHbEfuazvN9ipP2nr54kA-9-W448wts,7127
+biolib/_internal/data_record/data_record.py,sha256=brvr88Ri1ro-tHZ-_1S19mX5Yt4Tm7q1c5AoUZfpSNY,6990
 biolib/_internal/data_record/remote_storage_endpoint.py,sha256=LPq8Lr5FhKF9_o5K-bUdT7TeLe5XFUD0AAeTkNEVZug,1133
-biolib/_internal/http_client.py,sha256=cSBIzh00x2HL3BmLxt_GvDI7bWQMNytv_wOy7EXg8kI,4064
+biolib/_internal/http_client.py,sha256=L1pLe4OJv1YyoZJgRWbUtofBj-aovqJpawCuFFbVYDw,4470
 biolib/_internal/push_application.py,sha256=H1PGNtVJ0vRC0li39gFMpPpjm6QeZ8Ob-7cLkLmxS_Y,10009
 biolib/_internal/runtime.py,sha256=un18gmB2wFOXVjKca1Oe6mZI-xGydz8C8seScNvnC2s,2197
 biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
@@ -59,14 +59,14 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=ezXRhVBW5dYpLJjM2m5Qf-I6PrrNeBdEYjApjo7XVso,26419
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=n4BEn-bdL10uVvMDwBazK14NY25Fy_IhKY1fNRoX8LI,26455
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
 biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
 biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
-biolib/compute_node/job_worker/job_storage.py,sha256=Ol43f43W6aD2EUkA6G2i9-WxdREr5JPSjo1xFylddOQ,4030
-biolib/compute_node/job_worker/job_worker.py,sha256=QifGpDBn1ojJ5b43--XyXmz8YqsgNlGskh-z8mkYQqg,28147
+biolib/compute_node/job_worker/job_storage.py,sha256=LNkklckDLbYgCHsK5FGrEK75Kw-H4f4JcTCAtuE9His,4035
+biolib/compute_node/job_worker/job_worker.py,sha256=LH8URvDXHa2TT9yk5zUN0RKJGnf4h7WMVx70yQ8GniI,28986
 biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
@@ -105,8 +105,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=jC5WhH63FTD7SLFJbxQGA2hOt9NTwq9zHl_BEec1Z0c,4907
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.1896.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.1896.dist-info/METADATA,sha256=_yVGo6mNI1U75uCSkFegjMgTtSmymqhwh-VzZP1885M,1508
-pybiolib-1.1.1896.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.1896.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.1896.dist-info/RECORD,,
+pybiolib-1.1.1907.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.1907.dist-info/METADATA,sha256=Y7GVDjw8XNDr0aJmJECOojLgtBZ6UfAvIgOPoXtsr08,1508
+pybiolib-1.1.1907.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.1.1907.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.1907.dist-info/RECORD,,