pybiolib 1.1.1942__py3-none-any.whl → 1.1.1957__py3-none-any.whl

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -1,44 +1,41 @@
+import io
 import json
+import os
+import re
+import shlex
 import subprocess
+import tarfile
 import tempfile
 import time
-import tarfile
 import zipfile
-import os
-import io
-import re
-import shlex
 from copy import copy
 from datetime import datetime
 
 import docker  # type: ignore
 import docker.types  # type: ignore
-
-from docker.errors import ImageNotFound, APIError  # type: ignore
+from docker.errors import APIError, ImageNotFound  # type: ignore
 from docker.models.containers import Container  # type: ignore
 
 from biolib import utils
-
 from biolib._internal.runtime import RuntimeJobDataDict
 from biolib.biolib_binary_format import ModuleInput, ModuleOutputV2
+from biolib.biolib_binary_format.file_in_container import FileInContainer
 from biolib.biolib_docker_client import BiolibDockerClient
-from biolib.biolib_errors import DockerContainerNotFoundDuringExecutionException, BioLibError
+from biolib.biolib_errors import BioLibError, DockerContainerNotFoundDuringExecutionException
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node import utils as compute_node_utils
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker.docker_image_cache import DockerImageCache
-from biolib.biolib_binary_format.file_in_container import FileInContainer
 from biolib.compute_node.job_worker.executors.docker_types import DockerDiffKind
-from biolib.compute_node.job_worker.executors.types import StatusUpdate, LocalExecutorOptions
+from biolib.compute_node.job_worker.executors.types import LocalExecutorOptions, StatusUpdate
 from biolib.compute_node.job_worker.mappings import Mappings, path_without_first_folder
-from biolib.compute_node.job_worker.utils import ComputeProcessException
 from biolib.compute_node.job_worker.utilization_reporter_thread import UtilizationReporterThread
+from biolib.compute_node.job_worker.utils import ComputeProcessException
 from biolib.compute_node.utils import SystemExceptionCodes
-from biolib.typing_utils import List, Dict, Optional
+from biolib.typing_utils import Dict, List, Optional
 
 
 class DockerExecutor:
-
     def __init__(self, options: LocalExecutorOptions) -> None:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
@@ -116,35 +113,49 @@ class DockerExecutor:
             except Exception:  # pylint: disable=broad-except
                 logger_no_user_data.error('DockerExecutor failed to clean up container')
 
-    def _pull(self):
-        try:
-            start_time = time.time()
-            if utils.IS_RUNNING_IN_CLOUD and not self._options['job'].get('federated_job_uuid'):
-                DockerImageCache().get(
-                    image_uri=self._absolute_image_uri,
-                    estimated_image_size_bytes=self._options['module']['estimated_image_size_bytes'],
-                    job_id=self._options['job']['public_id'],
-                )
-            else:
-                docker_client = BiolibDockerClient.get_docker_client()
-                try:
-                    docker_client.images.get(self._absolute_image_uri)
-                except ImageNotFound:
-                    job_uuid = self._options['job'].get('federated_job_uuid') or self._options['job']['public_id']
-                    docker_client.images.pull(
-                        self._absolute_image_uri,
-                        auth_config={'username': 'biolib', 'password': f',{job_uuid}'},
-                    )
+    def _pull(self) -> None:
+        retries = 3
+        last_error: Optional[Exception] = None
+        estimated_image_size_bytes = self._options['module']['estimated_image_size_bytes']
+        assert estimated_image_size_bytes is not None, 'No estimated image size'
 
-            logger_no_user_data.debug(f'Pulled image in: {time.time() - start_time}')
+        for retry_count in range(retries + 1):
+            if retry_count > 0:
+                logger_no_user_data.debug(f'Retrying Docker image pull of "{self._absolute_image_uri}"')
+                time.sleep(5 * retry_count)
+            try:
+                start_time = time.time()
+                if utils.IS_RUNNING_IN_CLOUD and not self._options['job'].get('federated_job_uuid'):
+                    DockerImageCache().get(
+                        image_uri=self._absolute_image_uri,
+                        estimated_image_size_bytes=estimated_image_size_bytes,
+                        job_id=self._options['job']['public_id'],
+                    )
+                else:
+                    docker_client = BiolibDockerClient.get_docker_client()
+                    try:
+                        docker_client.images.get(self._absolute_image_uri)
+                    except ImageNotFound:
+                        job_uuid = self._options['job'].get('federated_job_uuid') or self._options['job']['public_id']
+                        docker_client.images.pull(
+                            self._absolute_image_uri,
+                            auth_config={'username': 'biolib', 'password': f',{job_uuid}'},
+                        )
+
+                logger_no_user_data.debug(f'Pulled image in: {time.time() - start_time}')
+                return
+            except Exception as error:
+                logger_no_user_data.warning(
+                    f'Pull of Docker image "{self._absolute_image_uri}" returned error: {error}'
+                )
+                last_error = error
 
-        except Exception as exception:
-            raise ComputeProcessException(
-                exception,
-                SystemExceptionCodes.FAILED_TO_PULL_DOCKER_IMAGE.value,
-                self._send_system_exception,
-                may_contain_user_data=False
-            ) from exception
+        raise ComputeProcessException(
+            last_error or Exception('Retries exceeded: failed to pull Docker image'),
+            SystemExceptionCodes.FAILED_TO_PULL_DOCKER_IMAGE.value,
+            self._send_system_exception,
+            may_contain_user_data=False,
+        )
 
     def _execute_helper(self, module_input, module_output_path: str) -> None:
         job_uuid = self._options['job']['public_id']
@@ -155,10 +166,10 @@ class DockerExecutor:
             logger_no_user_data.debug(f'Job "{job_uuid}" starting utilization metrics reporter thread...')
             config = CloudUtils.get_webserver_config()
             node_auth_token = config['compute_node_info']['auth_token']  # pylint: disable=unsubscriptable-object
-            cloud_job = self._options["cloud_job"]
+            cloud_job = self._options['cloud_job']
             include_gpu_stats = False
             if cloud_job:
-                include_gpu_stats = cloud_job.get("reserved_gpu_count", 0) > 0
+                include_gpu_stats = cloud_job.get('reserved_gpu_count', 0) > 0
             UtilizationReporterThread(
                 container=self._container,
                 job_uuid=job_uuid,
@@ -213,7 +224,7 @@ class DockerExecutor:
             raise ComputeProcessException(
                 MemoryError(),
                 SystemExceptionCodes.OUT_OF_MEMORY.value,
-                self._send_system_exception
+                self._send_system_exception,
             )
 
         logger_no_user_data.debug(f'Docker container exited with code {exit_code} for {job_uuid}')
@@ -244,19 +255,16 @@ class DockerExecutor:
         for path_to_delete in [self._input_tar_path, self._runtime_tar_path]:
             if os.path.exists(path_to_delete):
                 os.remove(path_to_delete)
-        logger_no_user_data.debug(f"Deleted tars in: {time.time() - tar_time}")
+        logger_no_user_data.debug(f'Deleted tars in: {time.time() - tar_time}')
 
         container_time = time.time()
         if self._docker_container:
             self._docker_container.remove(force=True)
 
         if utils.IS_RUNNING_IN_CLOUD:
-            DockerImageCache().detach_job(
-                image_uri=self._absolute_image_uri,
-                job_id=self._options['job']['public_id']
-            )
+            DockerImageCache().detach_job(image_uri=self._absolute_image_uri, job_id=self._options['job']['public_id'])
 
-        logger_no_user_data.debug(f"Deleted compute container in: {time.time() - container_time}")
+        logger_no_user_data.debug(f'Deleted compute container in: {time.time() - container_time}')
         self._tmp_secrets_dir.cleanup()
 
     # TODO: type this method
@@ -292,29 +300,35 @@ class DockerExecutor:
             )
             if app_version_created_at < datetime(2022, 11, 30, 0, 0):
                 environment_vars = module.get('secrets', {})
-                environment_vars.update({
-                    'BIOLIB_JOB_UUID': self._options['job']['public_id'],
-                    'BIOLIB_JOB_AUTH_TOKEN': self._options['job']['auth_token']
-                })
+                environment_vars.update(
+                    {
+                        'BIOLIB_JOB_UUID': self._options['job']['public_id'],
+                        'BIOLIB_JOB_AUTH_TOKEN': self._options['job']['auth_token'],
+                    }
+                )
 
             if utils.IS_RUNNING_IN_CLOUD and self._options['cloud_job']:
-                environment_vars.update({
-                    'BIOLIB_JOB_MAX_RUNTIME_IN_SECONDS': self._options['cloud_job']['max_runtime_in_seconds'],
-                })
+                environment_vars.update(
+                    {
+                        'BIOLIB_JOB_MAX_RUNTIME_IN_SECONDS': self._options['cloud_job']['max_runtime_in_seconds'],
+                    }
+                )
 
             for proxy in self._options['remote_host_proxies']:
                 proxy_ip = proxy.get_ip_address_on_network(internal_network)
                 if proxy.is_app_caller_proxy:
                     logger_no_user_data.debug('Found app caller proxy, setting both base URLs in compute container')
-                    environment_vars.update({
-                        'BIOLIB_BASE_URL': f'http://{proxy_ip}',
-                        'BIOLIB_CLOUD_BASE_URL': f'http://{proxy_ip}',
-                        # This should be removed eventually, but will break apps calling apps on older versions
-                        'BIOLIB_CLOUD_RESULTS_BASE_URL': f'http://{proxy_ip}',
-                        'BIOLIB_CLOUD_JOB_STORAGE_BASE_URL': f'http://{proxy_ip}',
-                        # Inform container if we are targeting public biolib as we change the BIOLIB_BASE_URL
-                        'BIOLIB_ENVIRONMENT_IS_PUBLIC_BIOLIB': bool(utils.BASE_URL_IS_PUBLIC_BIOLIB)
-                    })
+                    environment_vars.update(
+                        {
+                            'BIOLIB_BASE_URL': f'http://{proxy_ip}',
+                            'BIOLIB_CLOUD_BASE_URL': f'http://{proxy_ip}',
+                            # This should be removed eventually, but will break apps calling apps on older versions
+                            'BIOLIB_CLOUD_RESULTS_BASE_URL': f'http://{proxy_ip}',
+                            'BIOLIB_CLOUD_JOB_STORAGE_BASE_URL': f'http://{proxy_ip}',
+                            # Inform container if we are targeting public biolib as we change the BIOLIB_BASE_URL
+                            'BIOLIB_ENVIRONMENT_IS_PUBLIC_BIOLIB': bool(utils.BASE_URL_IS_PUBLIC_BIOLIB),
+                        }
+                    )
                 else:
                     extra_hosts[proxy.hostname] = proxy_ip
 
@@ -329,45 +343,36 @@ class DockerExecutor:
 
             if self._options['job'].get('arguments_override_command'):
                 # In this case, arguments contains a user specified command to run in the app
-                create_container_args.update({
-                    'command': module_input['arguments'],
-                    'entrypoint': ''
-                })
+                create_container_args.update({'command': module_input['arguments'], 'entrypoint': ''})
 
             else:
-                create_container_args.update({
-                    'command': shlex.split(module['command']) + module_input['arguments']
-                })
+                create_container_args.update({'command': shlex.split(module['command']) + module_input['arguments']})
 
             app_version = self._options['job']['app_version']
             if app_version.get('main_output_file') or app_version.get('stdout_render_type') == 'text':
                 create_container_args['tty'] = True
 
             if utils.IS_RUNNING_IN_CLOUD:
-                cloud_job = self._options["cloud_job"]
+                cloud_job = self._options['cloud_job']
                 create_container_args['mem_limit'] = f'{cloud_job["reserved_memory_in_bytes"]}b'
-                create_container_args['nano_cpus'] = cloud_job["reserved_cpu_in_nano_shares"]
+                create_container_args['nano_cpus'] = cloud_job['reserved_cpu_in_nano_shares']
 
                 biolib_identity_user_email: Optional[str] = cloud_job.get('biolib_identity_user_email')
                 if biolib_identity_user_email:
-                    create_container_args['environment'].update({
-                        'BIOLIB_IDENTITY_USER_EMAIL': biolib_identity_user_email
-                    })
+                    create_container_args['environment'].update(
+                        {'BIOLIB_IDENTITY_USER_EMAIL': biolib_identity_user_email}
+                    )
 
             docker_runtime = os.getenv('BIOLIB_DOCKER_RUNTIME')
             if docker_runtime is not None:
                 create_container_args['runtime'] = docker_runtime
 
-            self._docker_container = BiolibDockerClient.get_docker_client().containers.create(
-                **create_container_args
-            )
+            self._docker_container = BiolibDockerClient.get_docker_client().containers.create(**create_container_args)
 
             logger_no_user_data.debug('Finished initializing docker container')
         except Exception as exception:
             raise ComputeProcessException(
-                exception,
-                SystemExceptionCodes.FAILED_TO_START_COMPUTE_CONTAINER.value,
-                self._send_system_exception
+                exception, SystemExceptionCodes.FAILED_TO_START_COMPUTE_CONTAINER.value, self._send_system_exception
             ) from exception
 
     def _add_file_to_tar(self, tar, current_path, mapped_path, data):
@@ -434,7 +439,7 @@ class DockerExecutor:
             raise ComputeProcessException(
                 exception,
                 SystemExceptionCodes.FAILED_TO_COPY_INPUT_FILES_TO_COMPUTE_CONTAINER.value,
-                self._send_system_exception
+                self._send_system_exception,
             ) from exception
 
     def _map_and_copy_runtime_files_to_container(self, runtime_zip_data, arguments: List[str], remove_root_folder=True):
@@ -449,17 +454,17 @@ class DockerExecutor:
             raise ComputeProcessException(
                 exception,
                 SystemExceptionCodes.FAILED_TO_COPY_RUNTIME_FILES_TO_COMPUTE_CONTAINER.value,
-                self._send_system_exception
+                self._send_system_exception,
             ) from exception
 
     def _write_module_output_to_file(
-            self,
-            arguments: List[str],
-            exit_code: int,
-            module_output_path: str,
-            stderr: bytes,
-            stdout: bytes,
-            pre_start_diff: List[Dict],
+        self,
+        arguments: List[str],
+        exit_code: int,
+        module_output_path: str,
+        stderr: bytes,
+        stdout: bytes,
+        pre_start_diff: List[Dict],
     ) -> None:
         mapped_files: List[FileInContainer] = []
         try:
@@ -504,9 +509,11 @@ class DockerExecutor:
             result = subprocess.run(
                 args=[
                     'ctr',
-                    '--namespace', 'moby',
+                    '--namespace',
+                    'moby',
                     'snapshots',
-                    '--snapshotter', 'nydus',
+                    '--snapshotter',
+                    'nydus',
                     'mounts',
                     '/some_arbitrary_path',
                     str(self._container.id),
@@ -528,9 +535,10 @@ class DockerExecutor:
         pre_start_diff_paths = [obj['Path'] for obj in pre_start_diff]
         post_run_diff = self._container.diff()
         run_diff_paths: List[str] = [
-            obj['Path'] for obj in post_run_diff if
-            obj['Kind'] in (DockerDiffKind.CHANGED.value, DockerDiffKind.ADDED.value) and
-            obj['Path'] not in pre_start_diff_paths
+            obj['Path']
+            for obj in post_run_diff
+            if obj['Kind'] in (DockerDiffKind.CHANGED.value, DockerDiffKind.ADDED.value)
+            and obj['Path'] not in pre_start_diff_paths
         ]
 
         known_directories = set()
@@ -540,7 +548,7 @@ class DockerExecutor:
                 if idx == 0:
                     continue  # always skip root
 
-                folder = '/' + '/'.join(parent_folders[1:idx + 1])
+                folder = '/' + '/'.join(parent_folders[1 : idx + 1])
                 known_directories.add(folder)
 
         def path_is_included_in_from_mappings(path: str) -> bool:
@@ -560,11 +568,13 @@ class DockerExecutor:
         files_and_empty_dirs: List[FileInContainer] = []
         for path in run_diff_paths:
             if path not in known_directories and path_is_included_in_from_mappings(path):
-                files_and_empty_dirs.append(FileInContainer(
-                    container=self._container,
-                    overlay_upper_dir_path=overlay_upper_dir_path,
-                    path_in_container=path,
-                ))
+                files_and_empty_dirs.append(
+                    FileInContainer(
+                        container=self._container,
+                        overlay_upper_dir_path=overlay_upper_dir_path,
+                        path_in_container=path,
+                    )
+                )
 
         return files_and_empty_dirs
 

{pybiolib-1.1.1942.dist-info → pybiolib-1.1.1957.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1942
+Version: 1.1.1957
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.1942.dist-info → pybiolib-1.1.1957.dist-info}/RECORD

@@ -59,7 +59,7 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=Mh1Vnamq68iE_Vvd4WlrmY0CnDVW_4QTdVJoTJJyGsE,26512
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=Lo7qgOm6uolNyH7QIuLqTY_6RLyPYeAlOk9HzC82U-s,27332
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
@@ -105,8 +105,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=jC5WhH63FTD7SLFJbxQGA2hOt9NTwq9zHl_BEec1Z0c,4907
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.1942.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.1942.dist-info/METADATA,sha256=Pwe_8XEyKXVlgndcOctndELApx5t-8KLDuAc3EJFDis,1508
-pybiolib-1.1.1942.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.1942.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.1942.dist-info/RECORD,,
+pybiolib-1.1.1957.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.1.1957.dist-info/METADATA,sha256=dzeKnljFiNgqOp-j8BxQxOYZT_VHpj7yUabW8-HwR_k,1508
+pybiolib-1.1.1957.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.1.1957.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.1.1957.dist-info/RECORD,,