pybiolib 1.2.221__py3-none-any.whl → 1.2.230__py3-none-any.whl

biolib/_runtime/runtime.py

@@ -48,6 +48,17 @@ class Runtime:
         except BaseException as error:
             raise BioLibRuntimeError(f'Unable to get system secret: {secret_name}') from error
 
+    @staticmethod
+    def get_temporary_client_secret(secret_name: str) -> bytes:
+        assert re.match(
+            '^[a-zA-Z0-9_-]*$', secret_name
+        ), 'Secret name can only contain alphanumeric characters and dashes or underscores '
+        try:
+            with open(f'/biolib/temporary-client-secrets/{secret_name}', 'rb') as file:
+                return file.read()
+        except BaseException as error:
+            raise BioLibRuntimeError(f'Unable to get secret: {secret_name}') from error
+
     @staticmethod
     def set_main_result_prefix(result_prefix: str) -> None:
         job_data = Runtime._get_job_data()

biolib/biolib_api_client/job_types.py

@@ -2,7 +2,7 @@ from enum import Enum
 
 from biolib.biolib_api_client.app_types import AppVersionOnJob, RemoteHost
 from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import List, Optional, TypedDict
+from biolib.typing_utils import Dict, List, Optional, TypedDict
 
 
 class JobState(Enum):
@@ -31,6 +31,7 @@ class _Job(TypedDict):
 # type optional keys with total=False
 class CreatedJobDict(_Job, total=False):
     custom_compute_node_url: str
+    temporary_client_secrets: Dict[str, str]
 
 
 class CloudJob(TypedDict):

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -69,7 +69,9 @@ class DockerExecutor:
 
         # If BIOLIB_SECRETS_TMPFS_PATH is set create the temporary directory there
         self._tmp_secrets_dir = tempfile.TemporaryDirectory(dir=utils.BIOLIB_SECRETS_TMPFS_PATH or None)
+        self._tmp_client_secrets_dir = tempfile.TemporaryDirectory(dir=utils.BIOLIB_SECRETS_TMPFS_PATH or None)
         os.chmod(self._tmp_secrets_dir.name, 0o755)
+        os.chmod(self._tmp_client_secrets_dir.name, 0o755)
 
     @property
     def _container(self) -> Container:
@@ -263,6 +265,7 @@ class DockerExecutor:
 
         logger_no_user_data.debug(f'Deleted compute container in: {time.time() - container_time}')
         self._tmp_secrets_dir.cleanup()
+        self._tmp_client_secrets_dir.cleanup()
 
     # TODO: type this method
     def _initialize_docker_container(self, module_input):
@@ -285,11 +288,23 @@ class DockerExecutor:
                 app_uri=self._options['job']['app_uri'],
                 is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
             )
-            secrets: Dict[str, str] = dict(
-                **module.get('secrets', {}),
-                biolib_system_secret=json.dumps(biolib_system_secret, indent=4),
+            docker_volume_mounts.append(
+                self._create_secrets_mount(
+                    source_dir=self._tmp_secrets_dir.name + '/',
+                    target_dir='/biolib/secrets/',
+                    secrets=dict(
+                        **module.get('secrets', {}),
+                        biolib_system_secret=json.dumps(biolib_system_secret, indent=4),
+                    ),
+                )
+            )
+            docker_volume_mounts.append(
+                self._create_secrets_mount(
+                    source_dir=self._tmp_client_secrets_dir.name + '/',
+                    target_dir='/biolib/temporary-client-secrets/',
+                    secrets=self._options['job'].get('temporary_client_secrets', {}),
+                )
             )
-            docker_volume_mounts.append(self._create_secrets_mount(secrets))
 
             environment_vars = {}
 
@@ -498,8 +513,8 @@ class DockerExecutor:
                 stdout=stdout,
             )
         except Exception as exception:
-            logger.error("Hit Error 23")
-            logger.exception("Error")
+            logger.error('Hit Error 23')
+            logger.exception('Error')
             logger.error(str(exception))
             time.sleep(3)
             raise ComputeProcessException(
@@ -586,19 +601,16 @@ class DockerExecutor:
 
         return files_and_empty_dirs
 
-    def _create_secrets_mount(self, secrets: Dict[str, str]) -> docker.types.Mount:
-        job_uuid = self._options['job']['public_id']
+    def _create_secrets_mount(self, source_dir: str, target_dir: str, secrets: Dict[str, str]) -> docker.types.Mount:
+        assert source_dir.startswith('/') and source_dir.endswith('/'), 'source_dir must start and end with slash'
+        assert target_dir.startswith('/') and target_dir.endswith('/'), 'target_dir must start and end with slash'
 
+        job_uuid = self._options['job']['public_id']
         for key, value in secrets.items():
             if re.match(r'^[a-zA-Z0-9-_]+$', key):
-                with open(f'{self._tmp_secrets_dir.name}/{key}', 'w') as secret_file:
+                with open(f'{source_dir}{key}', 'w') as secret_file:
                     secret_file.write(value)
             else:
                 logger_no_user_data.warning(f'Job {job_uuid} uses secret with a key not matching validation regex')
 
-        return docker.types.Mount(
-            read_only=True,
-            source=f'{self._tmp_secrets_dir.name}/',
-            target='/biolib/secrets/',
-            type='bind',
-        )
+        return docker.types.Mount(read_only=True, source=source_dir, target=target_dir, type='bind')

biolib/compute_node/utils.py

@@ -1,6 +1,6 @@
-from enum import Enum
 import random
 import string
+from enum import Enum
 
 from biolib.biolib_logging import logger
 
@@ -29,7 +29,7 @@ def get_package_type(package):
         return 'StdoutAndStderr'
 
     else:
-        raise Exception(f"Unexpected package type {package_type}")
+        raise Exception(f'Unexpected package type {package_type}')
 
 
 class SystemExceptionCodes(Enum):
@@ -68,6 +68,8 @@ class SystemExceptionCodes(Enum):
     FAILED_TO_ALLOCATE_JOB_TO_COMPUTE_NODE = 32
     NO_MODULES_FOUND_ON_JOB = 33
     FAILED_TO_INITIALIZE_DOCKER_EXECUTOR = 34
+    COMPUTE_NODE_SHUTDOWN = 35
+    COMPUTE_NODE_SHUTDOWN_DUE_TO_HEALTH_CHECK_FAILURE = 36
 
 
 SystemExceptionCodeMap = {
@@ -106,6 +108,8 @@ SystemExceptionCodeMap = {
     32: 'Failed to allocate job to compute node',
     33: 'No modules found on job',
     34: 'Failed to initialize docker executor',
+    35: 'Compute node shutdown',
+    36: 'Compute node shutdown due to health check failure',
 }
 
 

{pybiolib-1.2.221.dist-info → pybiolib-1.2.230.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.2.221
+Version: 1.2.230
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.2.221.dist-info → pybiolib-1.2.230.dist-info}/RECORD

@@ -27,7 +27,7 @@ biolib/_internal/types/resource_version.py,sha256=sLxViYXloDDUhTDFgjegiQCj097OM1
 biolib/_internal/types/typing.py,sha256=D4EKKEe7kDx0K6lJi-H_XLtk-8w6nu2fdqn9bvzI-Xo,288
 biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
 biolib/_internal/utils/multinode.py,sha256=zWrQhcVK5u_xdWX2oIM-D_2fINqNPlqF_h71fu4K8LY,8279
-biolib/_runtime/runtime.py,sha256=b6AnHtfJ2DmEspC02sOGLt7VjienFKWaiR2PmMKnC38,3931
+biolib/_runtime/runtime.py,sha256=bZQ0m39R9jOBVAtlyvzDnOobKueOAQUCwMUZjDQnO7E,4439
 biolib/api/__init__.py,sha256=mQ4u8FijqyLzjYMezMUUbbBGNB3iFmkNdjXnWPZ7Jlw,138
 biolib/api/client.py,sha256=FRpdH5aI187b_I_4HUNi680v4iOP65z5f2RcUo8D8MA,3559
 biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
@@ -40,7 +40,7 @@ biolib/biolib_api_client/auth.py,sha256=kjm0ZHnH3I8so3su2sZbBxNHYp-ZUdrZ5lwQ0K36
 biolib/biolib_api_client/biolib_app_api.py,sha256=i5KPZNpopU7cqcjVfICml-31ca-1-CU_rdZgKiZt0Vs,4343
 biolib/biolib_api_client/biolib_job_api.py,sha256=7bKfav3-12ewXkEUoLdCmbWdebW8148kxfGJW9SsXZI,7125
 biolib/biolib_api_client/common_types.py,sha256=RH-1KNHqUF-EkTpfPOSTt5Mq1GPdfju_cqXDesscO1I,123
-biolib/biolib_api_client/job_types.py,sha256=Dl4NhU2xpgpXV-7YIoDf6WL63SLR5bni55OX8x5539M,1300
+biolib/biolib_api_client/job_types.py,sha256=yBdBwjharbQJuXCi2xKMi0t_r6XxnbWnkchHekTpCJY,1351
 biolib/biolib_api_client/lfs_types.py,sha256=joZWP6-sa5_Ug_6xIp5fHAgEo_bqLE3rbleQocZtDcg,339
 biolib/biolib_api_client/user_state.py,sha256=XcgWV-MgVk88mIlMmnu8yHxMu8OCaw8o0tk7TVo5Hcg,637
 biolib/biolib_binary_format/__init__.py,sha256=HMl5SdX_VUWE4OQzi4Jf_yFvC7b0bSPOGPHYi9dWM2Q,185
@@ -78,7 +78,7 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=Rt_kpqpBmn6Ut4h-46nqc-g4Qhs4gtbNwEpLiOKOFOM,27939
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=HiAenSATWY7DApAKSBQU74jvA-JF42x5dBV8JxQ_eQ0,28829
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
@@ -93,7 +93,7 @@ biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO
 biolib/compute_node/remote_host_proxy.py,sha256=1EZnB0WWG359Yy22xBoqRqGEmmlA12rpiG5u8B2381M,16533
 biolib/compute_node/socker_listener_thread.py,sha256=T5_UikA3MB9bD5W_dckYLPTgixh72vKUlgbBvj9dbM0,1601
 biolib/compute_node/socket_sender_thread.py,sha256=YgamPHeUm2GjMFGx8qk-99WlZhEs-kAb3q_2O6qByig,971
-biolib/compute_node/utils.py,sha256=M7i_WTyxbFM3Lri9RWZ_8FeQNYrQIWpKGLfp2I55oeY,4677
+biolib/compute_node/utils.py,sha256=fWtFIcGatek1NaVOT9Vaey6UFp8GKLYGLAVu8jgOwi8,4861
 biolib/compute_node/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLUXWo_vHFV-FIwlb8s8tacKmGHvh93qc,914
 biolib/compute_node/webserver/webserver.py,sha256=o4kOAStsqThUtKlnRE-U5TP0JIYntuySDjU7PH310xg,6620
@@ -120,8 +120,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=Ozk0blGtPur_D9MwShD02r_mphyQmgZkx-lOHOwnlIM,6730
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.221.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.221.dist-info/METADATA,sha256=8NVC3cqB7FxyvdBVvQdS0uwnAYGGFfMIzyTssOX9CHk,1558
-pybiolib-1.2.221.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-pybiolib-1.2.221.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.221.dist-info/RECORD,,
+pybiolib-1.2.230.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.230.dist-info/METADATA,sha256=pHStJR-mwTW8Plixvbbd1rtb8Msj8BmiCBOi6-ORvnY,1558
+pybiolib-1.2.230.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+pybiolib-1.2.230.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.230.dist-info/RECORD,,