pybiolib 1.2.883__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/cli/run.py

@@ -13,19 +13,23 @@ from biolib.typing_utils import Optional, Tuple
     help='Run an application on BioLib.',
 )
 @click.option('--experiment', type=str, required=False, help='Experiment name or URI to add the run to.')
-@click.option('--local', is_flag=True, required=False, help='Run the application locally.')
+@click.option('--local', is_flag=True, required=False, hidden=True)
 @click.option('--non-blocking', is_flag=True, required=False, help='Run the application non blocking.')
 @click.argument('uri', required=True)
 @click.argument('args', nargs=-1, type=click.UNPROCESSED)
 def run(experiment: Optional[str], local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
+    if local:
+        print('Error: Running applications locally with --local is no longer supported.', file=sys.stderr)
+        sys.exit(1)
+
     if experiment:
         with Experiment(uri=experiment):
-            _run(local=local, non_blocking=non_blocking, uri=uri, args=args)
+            _run(non_blocking=non_blocking, uri=uri, args=args)
     else:
-        _run(local=local, non_blocking=non_blocking, uri=uri, args=args)
+        _run(non_blocking=non_blocking, uri=uri, args=args)
 
 
-def _run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
+def _run(non_blocking: bool, uri: str, args: Tuple[str]) -> None:
     try:
         app = BioLibApp(uri=uri)
     except biolib_errors.BioLibError as error:
@@ -43,12 +47,11 @@ def _run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
         args=list(args),
         stdin=_get_stdin(),
         files=None,
-        machine=('local' if local else ''),
         blocking=blocking,
     )
 
     if blocking:
-        job.save_files('biolib_results')
+        job.save_files('biolib_results', overwrite=True)
 
         # Write stdout and stderr if it has not been streamed (Markdown is not streamed)
         if app.version.get('stdout_render_type') == 'markdown' or not sys.stdout.isatty():

biolib/cli/start.py

@@ -13,7 +13,10 @@ from biolib.typing_utils import Optional
 @click.option('--port', default=5000, type=click.IntRange(1, 65_535), required=False)
 @click.option('--tls-certificate', type=click.Path(exists=True), required=False, hidden=True)
 @click.option('--tls-key', type=click.Path(exists=True), required=False, hidden=True)
-def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str]) -> None:
+@click.option('--initialize-network-buffer', is_flag=True, help='Initialize the remote host network buffer and exit')
+def start(
+    host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str], initialize_network_buffer: bool
+) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     if platform.system() == 'Windows':
@@ -22,6 +25,16 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
     if tls_certificate and not tls_key or tls_key and not tls_certificate:
         raise Exception('Options --tls-certificate and --tls-key must be specified together')
 
+    if initialize_network_buffer:
+        from biolib.compute_node.job_worker.network_buffer import (  # pylint: disable=import-outside-toplevel
+            NetworkBuffer,
+        )
+
+        network_buffer = NetworkBuffer.get_instance()
+        created = network_buffer.fill_buffer()
+        logger_no_user_data.info(f'Initialized network buffer (created {created} networks)')
+        return
+
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
 

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -11,10 +11,10 @@ import zipfile
 from copy import copy
 from datetime import datetime
 
-import docker  # type: ignore
-import docker.types  # type: ignore
-from docker.errors import APIError, ImageNotFound  # type: ignore
-from docker.models.containers import Container  # type: ignore
+import docker
+import docker.types
+from docker.errors import APIError, ImageNotFound
+from docker.models.containers import Container
 
 from biolib import utils
 from biolib._internal.runtime import RuntimeJobDataDict
@@ -40,7 +40,7 @@ class DockerExecutor:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
-        self._absolute_image_uri = f"{utils.BIOLIB_SITE_HOSTNAME}/{self._options['module']['image_uri']}"
+        self._absolute_image_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{self._options["module"]["image_uri"]}'
         self._send_system_exception = options['send_system_exception']
         self._send_stdout_and_stderr = options['send_stdout_and_stderr']
         self._random_docker_id = compute_node_utils.random_string(15)
@@ -308,7 +308,8 @@ class DockerExecutor:
             job_uuid = self._options['job']['public_id']
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container...')
             module = self._options['module']
-            logger.debug(f"Initializing docker container with command: {module['command']}")
+            logger.debug(f'Initializing docker container with command: {module["command"]}')
+            docker_client = BiolibDockerClient.get_docker_client()
 
             docker_volume_mounts = [lfs.docker_mount for lfs in self._options['large_file_systems'].values()]
 
@@ -318,10 +319,12 @@ class DockerExecutor:
             biolib_system_secret = RuntimeJobDataDict(
                 version='1.0.0',
                 job_requested_machine=self._options['job']['requested_machine'],
+                job_requested_machine_spot=self._options['job'].get('requested_machine_spot', False),
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
                 is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
+                job_reserved_machines=self._options['job']['reserved_machines'],
             )
             docker_volume_mounts.append(
                 self._create_secrets_mount(
@@ -365,9 +368,11 @@ class DockerExecutor:
                 )
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Getting IPs for proxies...')
+
+            networks_to_connect = []
             for proxy in self._options['remote_host_proxies']:
-                proxy_ip = proxy.get_ip_address_on_network(internal_network)
                 if proxy.is_app_caller_proxy:
+                    proxy_ip = proxy.get_ip_address_on_network(internal_network)
                     logger_no_user_data.debug('Found app caller proxy, setting both base URLs in compute container')
                     environment_vars.update(
                         {
@@ -381,7 +386,11 @@ class DockerExecutor:
                         }
                     )
                 else:
-                    extra_hosts[proxy.hostname] = proxy_ip
+                    extra_hosts.update(proxy.get_hostname_to_ip_mapping())
+
+                for network in proxy.get_remote_host_networks():
+                    if network != internal_network:
+                        networks_to_connect.append(network)
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Constructing container args...')
             create_container_args = {
@@ -391,6 +400,9 @@ class DockerExecutor:
                 'mounts': docker_volume_mounts,
                 'network': internal_network.name,
                 'working_dir': module['working_directory'],
+                'networking_config': {
+                    internal_network.name: docker_client.api.create_endpoint_config(aliases=['main'])
+                },
             }
 
             if self._options['job'].get('arguments_override_command'):
@@ -429,9 +441,19 @@ class DockerExecutor:
             if docker_runtime is not None:
                 create_container_args['runtime'] = docker_runtime
 
-            docker_client = BiolibDockerClient.get_docker_client()
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Creating container...')
             self._docker_container = docker_client.containers.create(**create_container_args)
+
+            if networks_to_connect:
+                network_connection_start = time.time()
+                for network in networks_to_connect:
+                    network.connect(self._docker_container.id)
+                    logger_no_user_data.debug(f'Connected app container to network {network.name}')
+                network_connection_time = time.time() - network_connection_start
+                logger_no_user_data.debug(
+                    f'Connected app container to {len(networks_to_connect)} networks in {network_connection_time:.2f}s'
+                )
+
             logger_no_user_data.debug(f'Job "{job_uuid}" finished initializing Docker container.')
         except Exception as exception:
             raise ComputeProcessException(

biolib/compute_node/job_worker/executors/docker_types.py

@@ -1,6 +1,6 @@
 import enum
 
-from biolib.typing_utils import TypedDict, Any
+from biolib.typing_utils import Any, TypedDict
 
 
 class Proxy(TypedDict):

biolib/compute_node/job_worker/executors/types.py

@@ -1,11 +1,11 @@
-from docker.models.networks import Network  # type: ignore
+from docker.models.networks import Network
 
+from biolib.biolib_api_client.app_types import Module
+from biolib.biolib_api_client.job_types import CloudJob, CreatedJobDict
 from biolib.compute_node.job_worker.large_file_system import LargeFileSystem
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Callable, Optional, List, Dict
 from biolib.compute_node.remote_host_proxy import RemoteHostProxy
-from biolib.biolib_api_client.app_types import Module
-from biolib.biolib_api_client.job_types import CreatedJobDict, CloudJob
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Callable, Dict, List, Optional, TypedDict
 
 
 class StatusUpdate(TypedDict):
@@ -43,6 +43,7 @@ class LocalExecutorOptions(TypedDict):
     send_system_exception: SendSystemExceptionType
     send_stdout_and_stderr: SendStdoutAndStderrType
 
+
 class MetadataToSaveOutput(TypedDict):
     arguments: List[str]
     startup_error_string: Optional[str]

biolib/compute_node/job_worker/job_storage.py

@@ -97,11 +97,12 @@ class JobStorage:
     @staticmethod
     def download_module_input(job: CreatedJobDict, path: str):
         job_uuid = job['public_id']
-        logger_no_user_data.debug(f'Job "{job_uuid}" downloading module input...')
+        logger_no_user_data.debug(f'Job "{job_uuid}" getting module input url...')
         presigned_download_url = BiolibJobApi.get_job_storage_download_url(
             job_uuid=job_uuid,
             job_auth_token=job['auth_token'],
             storage_type='input',
         )
+        logger_no_user_data.debug(f'Job "{job_uuid}" downloading module input...')
         HttpClient.request(url=presigned_download_url, response_path=path)
         logger_no_user_data.debug(f'Job "{job_uuid}" module input downloaded')