pybiolib 1.2.911__py3-none-any.whl → 1.2.1642__py3-none-any.whl

biolib/biolib_api_client/biolib_app_api.py

@@ -57,6 +57,22 @@ def _get_git_branch_name() -> str:
         return ''
 
 
+def _get_git_commit_hash() -> str:
+    try:
+        github_actions_commit_hash = os.getenv('GITHUB_SHA')
+        if github_actions_commit_hash:
+            return github_actions_commit_hash
+
+        gitlab_ci_commit_hash = os.getenv('CI_COMMIT_SHA')
+        if gitlab_ci_commit_hash:
+            return gitlab_ci_commit_hash
+
+        result = subprocess.run(['git', 'rev-parse', 'HEAD'], check=True, stdout=subprocess.PIPE, text=True)
+        return result.stdout.strip()
+    except BaseException:
+        return ''
+
+
 def _get_git_repository_url() -> str:
     try:
         result = subprocess.run(['git', 'remote', 'get-url', 'origin'], check=True, stdout=subprocess.PIPE, text=True)
@@ -99,6 +115,15 @@ class BiolibAppApi:
 
             raise error
 
+    @staticmethod
+    def create_app(uri: str):
+        uri = _get_app_uri_from_str(uri)
+        try:
+            response = biolib.api.client.post(path='/resources/apps/', data={'uri': uri})
+            return response.json()
+        except HttpError as error:
+            raise error
+
     @staticmethod
     def push_app_version(
         app_id,
@@ -116,6 +141,7 @@ class BiolibAppApi:
                 'state': 'published',
                 'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
                 'git_branch_name': _get_git_branch_name(),
+                'git_commit_hash': _get_git_commit_hash(),
                 'git_repository_url': _get_git_repository_url(),
             }
             if semantic_version:

biolib/biolib_binary_format/module_input.py

@@ -1,4 +1,5 @@
 from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
+from biolib.biolib_logging import logger
 from biolib.typing_utils import TypedDict, Dict, List
 
 
@@ -14,6 +15,10 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
         self.package_type = 1
 
     def serialize(self, stdin, arguments, files) -> bytes:
+        for path in files.keys():
+            if '//' in path:
+                raise ValueError(f"File path '{path}' contains double slashes which are not allowed")
+
         bbf_data = bytearray()
         bbf_data.extend(self.version.to_bytes(1, 'big'))
         bbf_data.extend(self.package_type.to_bytes(1, 'big'))
@@ -67,6 +72,9 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
             data_len = self.get_data(8, output_type='int')
             path = self.get_data(path_len, output_type='str')
             data = self.get_data(data_len)
+            if '//' in path:
+                # TODO: Raise ValueError here once backwards compatibility period is over
+                logger.warning(f"File path '{path}' contains double slashes which are not allowed")
             files[path] = bytes(data)
 
         return ModuleInputDict(stdin=stdin, arguments=arguments, files=files)

biolib/biolib_binary_format/remote_endpoints.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format.utils import RemoteEndpoint
@@ -17,13 +17,13 @@ class RemoteJobStorageEndpoint(RemoteEndpoint):
         self._storage_type: Literal['input', 'output'] = storage_type
 
     def get_remote_url(self):
-        if not self._presigned_url or datetime.utcnow() > self._expires_at:
+        if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
             self._presigned_url = BiolibJobApi.get_job_storage_download_url(
                 job_auth_token=self._job_auth_token,
                 job_uuid=self._job_uuid,
                 storage_type='results' if self._storage_type == 'output' else 'input',
             )
-            self._expires_at = datetime.utcnow() + timedelta(minutes=8)
+            self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
             # TODO: Use expires at from url
             # parsed_url = urlparse(self._presigned_url)
             # query_params = parse_qs(parsed_url.query)

biolib/biolib_binary_format/remote_stream_seeker.py

@@ -1,45 +1,59 @@
 from biolib.biolib_binary_format.utils import IndexableBuffer
+from biolib.biolib_logging import logger
 from biolib.typing_utils import Iterable
 
 
 class StreamSeeker:
     def __init__(
-            self,
-            upstream_buffer: IndexableBuffer,
-            files_data_start: int,
-            files_data_end: int,
-            download_chunk_size_in_bytes: int,
+        self,
+        upstream_buffer: IndexableBuffer,
+        files_data_start: int,
+        files_data_end: int,
+        max_chunk_size: int,
     ):
         self._upstream_buffer = upstream_buffer
         self._files_data_end = files_data_end
-        self._download_chunk_size_in_bytes = download_chunk_size_in_bytes
+        self._max_chunk_size = max_chunk_size
 
         self._buffer_start = files_data_start
         self._buffer = bytearray()
 
-    def seek_and_read(self, file_start: int, file_length: int) -> Iterable[bytes]:
+    def seek_and_read(self, file_start: int, file_length: int, read_ahead_bytes: int = 0) -> Iterable[bytes]:
         assert file_start >= self._buffer_start
-        self._buffer = self._buffer[file_start - self._buffer_start:]  # Returns empty array if "out of bounds"
+        self._buffer = self._buffer[file_start - self._buffer_start :]
         self._buffer_start = file_start
 
         while True:
             file_byte_count_remaining = file_length - (self._buffer_start - file_start)
-            if file_byte_count_remaining == 0:
+            if file_byte_count_remaining <= 0:
                 return
 
-            start_of_fetch = self._buffer_start + len(self._buffer)
-            byte_count_left_in_stream = self._files_data_end - start_of_fetch
-
-            if byte_count_left_in_stream != 0:
-                # Only fetch if there is still data left upstream
-                if self._download_chunk_size_in_bytes > len(self._buffer):
-                    # Only fetch if size of buffer is below chunk size
-                    self._buffer.extend(self._upstream_buffer.get_data(
-                        start=start_of_fetch,
-                        length=min(byte_count_left_in_stream, self._download_chunk_size_in_bytes),
-                    ))
-
-            bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
-            yield bytes_to_yield
-            self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
-            self._buffer_start += len(bytes_to_yield)
+            if len(self._buffer) > 0:
+                take = min(file_byte_count_remaining, len(self._buffer))
+                chunk = self._buffer[:take]
+                if chunk:
+                    yield chunk
+                self._buffer = self._buffer[take:]
+                self._buffer_start += take
+            else:
+                start_of_fetch = self._buffer_start + len(self._buffer)
+                bytes_left_in_stream = self._files_data_end - start_of_fetch
+                if bytes_left_in_stream <= 0:
+                    logger.error(
+                        'StreamSeeker: no bytes left upstream (start_of_fetch=%d, files_data_end=%d)',
+                        start_of_fetch,
+                        self._files_data_end,
+                    )
+                    return
+
+                fetch_size = min(self._max_chunk_size, file_byte_count_remaining + read_ahead_bytes)
+                if fetch_size > bytes_left_in_stream:
+                    logger.error(
+                        'StreamSeeker: fetch_size (%d) > bytes_left_in_stream (%d); clamping',
+                        fetch_size,
+                        bytes_left_in_stream,
+                    )
+                    fetch_size = bytes_left_in_stream
+
+                fetched_data = self._upstream_buffer.get_data(start=start_of_fetch, length=fetch_size)
+                self._buffer.extend(fetched_data)

biolib/biolib_logging.py

@@ -62,7 +62,7 @@ def _get_no_user_data_logger() -> _BioLibLogger:
 
     # TODO: Simplify by refactoring to env BIOLIB_ENVIRONMENT_IS_CLOUD: boolean
     if os.getenv('BIOLIB_CLOUD_ENVIRONMENT', '').lower() == 'non-enclave':
-        handler = logging.FileHandler(filename='/tmp/biolib_no_user_data.log')
+        handler = logging.FileHandler(filename='/biolib/logs/biolib_no_user_data.log')
         formatter = logging.Formatter(_DEFAULT_LOGGER_FORMAT)
         handler.setFormatter(formatter)
         _logger_no_user_data.addHandler(handler)

biolib/cli/__init__.py

@@ -5,7 +5,7 @@ import click
 
 from biolib import utils
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.cli import auth, data_record, download_container, init, lfs, push, run, runtime, sdk, start
+from biolib.cli import auth, data_record, download_container, index, init, lfs, push, run, runtime, sdk, start
 
 
 @click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
@@ -31,6 +31,7 @@ cli.add_command(run.run)
 cli.add_command(runtime.runtime)
 cli.add_command(start.start)
 cli.add_command(data_record.data_record)
+cli.add_command(index.index)
 cli.add_command(sdk.sdk)
 
 # allow this script to be called without poetry in dev e.g. by an IDE debugger

biolib/cli/auth.py

@@ -3,7 +3,7 @@ import sys
 
 import click
 
-from biolib import api, biolib_errors
+from biolib import api
 from biolib.biolib_api_client.api_client import BiolibApiClient
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.user import sign_in, sign_out
@@ -35,23 +35,11 @@ def logout() -> None:
 def whoami() -> None:
     client = BiolibApiClient.get()
     if client.is_signed_in:
-        user_uuid = None
-        if client.access_token is None:
-            print('Unable to fetch user credentials. Please try logging out and logging in again.')
-            exit(1)
-        try:
-            user_uuid = client.decode_jwt_without_checking_signature(jwt=client.access_token)['payload']['public_id']
-        except biolib_errors.BioLibError as error:
-            print(
-                f'Unable to reference user public_id in access token:\n {error.message}',
-                file=sys.stderr,
-            )
-            exit(1)
-        response = api.client.get(path=f'/user/{user_uuid}/')
+        response = api.client.get(path='/users/me/')
         user_dict = response.json()
         email = user_dict['email']
-        intrinsic_account = [account for account in user_dict['accounts'] if account['role'] == 'intrinsic'][0]
-        display_name = intrinsic_account['display_name']
+        display_name = user_dict['account']['display_name']
+
         print(f'Name: {display_name}\nEmail: {email}\nLogged into: {client.base_url}')
     else:
         print('Not logged in', file=sys.stderr)

biolib/cli/data_record.py

@@ -80,3 +80,20 @@ def describe(uri: str, output_as_json: bool) -> None:
             size_string = str(file_info['size_bytes'])
             leading_space_string = ' ' * (10 - len(size_string))
             print(f"{leading_space_string}{size_string}    {file_info['path']}")
+
+
+@data_record.command(help='Delete a Data Record')
+@click.argument('uri', required=True)
+def delete(uri: str) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+
+    print(f'You are about to delete the data record: {record.uri}')
+    print('This action cannot be undone.')
+
+    confirmation = input(f'To confirm deletion, please type the data record name "{record.name}": ')
+    if confirmation != record.name:
+        print('Data record name does not match. Deletion cancelled.')
+        return
+
+    record.delete()
+    print(f'Data record {record.uri} has been deleted.')

biolib/cli/index.py

@@ -0,0 +1,32 @@
+import json
+import logging
+import sys
+
+import click
+
+from biolib._index.index import Index
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger, logger_no_user_data
+
+
+@click.group(help='Manage Indexes')
+def index() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@index.command(help='Create an Index')
+@click.argument('uri', required=True)
+@click.option('--config-path', required=True, type=click.Path(exists=True), help='Path to JSON config file')
+def create(uri: str, config_path: str) -> None:
+    try:
+        Index.create_from_config_file(uri=uri, config_path=config_path)
+    except json.JSONDecodeError as error:
+        print(f'Error: Invalid JSON in config file: {error}', file=sys.stderr)
+        sys.exit(1)
+    except BioLibError as error:
+        print(f'Error creating index: {error.message}', file=sys.stderr)
+        sys.exit(1)
+    except Exception as error:
+        print(f'Error reading config file: {error}', file=sys.stderr)
+        sys.exit(1)

biolib/cli/init.py

@@ -4,9 +4,19 @@ import sys
 
 import click
 
-from biolib import utils  # Import like this to let BASE_URL_IS_PUBLIC_BIOLIB be set correctly
+from biolib import (
+    biolib_errors,
+    utils,  # Import like this to let BASE_URL_IS_PUBLIC_BIOLIB be set correctly
+)
 from biolib._internal.add_copilot_prompts import add_copilot_prompts
+from biolib._internal.add_gui_files import add_gui_files
+from biolib._internal.http_client import HttpError
+from biolib._internal.string_utils import normalize_for_docker_tag
 from biolib._internal.templates import templates
+from biolib.api import client as api_client
+from biolib.biolib_api_client.api_client import BiolibApiClient
+from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.user.sign_in import sign_in
 from biolib.utils import BIOLIB_PACKAGE_VERSION
 
 
@@ -15,25 +25,82 @@ def init() -> None:
     cwd = os.getcwd()
 
     app_uri = input('What URI do you want to create the application under? (leave blank to skip): ')
+
+    if app_uri and not app_uri.startswith('@'):
+        try:
+            response = api_client.get('system/enterprise/config/', authenticate=False)
+            config = response.json()
+            prefix = config.get('resource_hostname_prefix')
+            if prefix:
+                app_uri = f'@{prefix}/{app_uri}'
+                print(f'Detected enterprise deployment, using URI: {app_uri}')
+        except HttpError as e:
+            # 404 indicates endpoint not found, 501 indicates non-enterprise deployment
+            if e.code in [404, 501]:
+                pass
+            else:
+                print(f'Warning: Could not detect enterprise configuration: {e}')
+        except Exception as e:
+            print(f'Warning: Could not detect enterprise configuration: {e}')
+
     app_name = app_uri.split('/')[-1] if app_uri else None
+    docker_tag = normalize_for_docker_tag(app_name) if app_name else None
+
+    if app_uri:
+        try:
+            if BiolibApiClient.is_reauthentication_needed():
+                sign_in_input = input('You need to sign in to validate/create apps. Would you like to sign in? [y/N]: ')
+                if sign_in_input.lower() in ['y', 'yes']:
+                    sign_in()
+                else:
+                    print('Skipping app validation and creation. You can set the URI in .biolib/config.yml later.')
+                    return
+
+            BiolibAppApi.get_by_uri(app_uri)
+            print(f'App {app_uri} already exists.')
+        except biolib_errors.NotFound:
+            create_app_input = input(f'App {app_uri} does not exist. Would you like to create it? [y/N]: ')
+            if create_app_input.lower() in ['y', 'yes']:
+                try:
+                    BiolibAppApi.create_app(app_uri)
+                    print(f'Successfully created app {app_uri}')
+                except Exception as e:
+                    print(f'Failed to create app {app_uri}: {str(e)}')
+                    print('You can create the app manually later or set the URI in .biolib/config.yml')
+            else:
+                print(
+                    'App creation skipped. You can create the app manually later or set the URI in .biolib/config.yml'
+                )
+        except Exception as e:
+            print(f'Failed to validate app {app_uri}: {str(e)}')
+            print('Continuing with initialization...')
+
     if not app_uri:
         print(
             'Remember to set the app URI in the .biolib/config.yml file later, '
             'and docker image name in the .biolib/config.yml and .github/workflows/biolib.yml files.'
         )
-    copilot_input = input('Do you want to include Copilot style prompts? [y/N]: ')
-    include_copilot_style = copilot_input.lower() == 'y'
-
-    template_dir = templates.init_template()
+    advanced_setup_input = input('Do you want to set up advanced features like Copilot and GUI? [y/N]: ')
+    advanced_setup = advanced_setup_input.lower() == 'y'
+    include_copilot = False
+    include_gui = False
+    if advanced_setup:
+        copilot_enabled_input = input('Do you want to include Copilot instructions and prompts? [y/N]: ')
+        include_copilot = copilot_enabled_input.lower() == 'y'
+        include_gui_input = input('Do you want to include GUI setup? [y/N]: ')
+        include_gui = include_gui_input.lower() == 'y'
+
+    init_template_dir = templates.init_template()
     conflicting_files = []
     files_to_overwrite = set()
 
     try:
         # First pass: check for conflicts
-        for root, dirs, filenames in os.walk(template_dir):
+        for root, dirs, filenames in os.walk(init_template_dir):
             dirs[:] = [d for d in dirs if '__pycache__' not in d]
-            relative_dir = os.path.relpath(root, template_dir)
+            relative_dir = os.path.relpath(root, init_template_dir)
             destination_dir = cwd if relative_dir == '.' else os.path.join(cwd, relative_dir)
+
             for filename in filenames:
                 source_file = os.path.join(root, filename)
                 destination_file = os.path.join(destination_dir, filename)
@@ -54,18 +121,22 @@ def init() -> None:
                     files_to_overwrite.add(conflicting_file)
 
         replace_app_uri = app_uri if app_uri else 'PUT_APP_URI_HERE'
+        replace_app_name = app_name if app_name else 'biolib-app'
 
         # Second pass: copy files (only if no conflicts)
-        for root, dirs, filenames in os.walk(template_dir):
+        for root, dirs, filenames in os.walk(init_template_dir):
             dirs[:] = [d for d in dirs if '__pycache__' not in d]
-            relative_dir = os.path.relpath(root, template_dir)
+            relative_dir = os.path.relpath(root, init_template_dir)
             destination_dir = os.path.join(cwd, relative_dir)
+
             os.makedirs(destination_dir, exist_ok=True)
 
             for filename in filenames:
                 if utils.BASE_URL_IS_PUBLIC_BIOLIB and filename == 'biolib.yml':
                     continue
 
+                relative_file_path = os.path.join(relative_dir, filename) if relative_dir != '.' else filename
+
                 source_file = os.path.join(root, filename)
                 destination_file = os.path.join(destination_dir, filename)
                 relative_file_path = os.path.relpath(destination_file, cwd)
@@ -79,8 +150,15 @@ def init() -> None:
                         new_content = new_content.replace('BIOLIB_REPLACE_APP_URI', replace_app_uri)
                         new_content = new_content.replace(
                             'BIOLIB_REPLACE_DOCKER_TAG',
-                            app_name if app_name else 'PUT_DOCKER_TAG_HERE',
+                            docker_tag if docker_tag else 'PUT_DOCKER_TAG_HERE',
                         )
+                        new_content = new_content.replace('BIOLIB_REPLACE_APP_NAME', replace_app_name)
+
+                        gui_config = "main_output_file: '/result.html'\n" if include_gui else ''
+                        new_content = new_content.replace('BIOLIB_REPLACE_GUI_CONFIG\n', gui_config)
+
+                        gui_mv_command = 'mv result.html output/result.html\n' if include_gui else ''
+                        new_content = new_content.replace('BIOLIB_REPLACE_GUI_MV_COMMAND\n', gui_mv_command)
 
                         with open(destination_file, 'w') as f:
                             f.write(new_content)
@@ -92,7 +170,11 @@ def init() -> None:
             with open(readme_path, 'w') as readme_file:
                 readme_file.write(f'# {app_name}\n')
 
-        add_copilot_prompts(force=False, style=include_copilot_style, silent=True)
+        if include_copilot:
+            add_copilot_prompts(force=False, silent=True)
+
+        if include_gui:
+            add_gui_files(force=False, silent=True)
 
     except KeyboardInterrupt:
         print('\nInit command cancelled.', file=sys.stderr)

biolib/cli/lfs.py

@@ -13,7 +13,7 @@ from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import Optional
 
 
-@click.group(help='Manage Large File Systems')
+@click.group(help='Manage Large File Systems', hidden=True)
 def lfs() -> None:
     pass
 

biolib/cli/run.py

@@ -48,7 +48,7 @@ def _run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
     )
 
     if blocking:
-        job.save_files('biolib_results')
+        job.save_files('biolib_results', overwrite=True)
 
         # Write stdout and stderr if it has not been streamed (Markdown is not streamed)
         if app.version.get('stdout_render_type') == 'markdown' or not sys.stdout.isatty():

biolib/cli/start.py

@@ -13,7 +13,10 @@ from biolib.typing_utils import Optional
 @click.option('--port', default=5000, type=click.IntRange(1, 65_535), required=False)
 @click.option('--tls-certificate', type=click.Path(exists=True), required=False, hidden=True)
 @click.option('--tls-key', type=click.Path(exists=True), required=False, hidden=True)
-def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str]) -> None:
+@click.option('--initialize-network-buffer', is_flag=True, help='Initialize the remote host network buffer and exit')
+def start(
+    host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str], initialize_network_buffer: bool
+) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     if platform.system() == 'Windows':
@@ -22,6 +25,16 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
     if tls_certificate and not tls_key or tls_key and not tls_certificate:
         raise Exception('Options --tls-certificate and --tls-key must be specified together')
 
+    if initialize_network_buffer:
+        from biolib.compute_node.job_worker.network_buffer import (  # pylint: disable=import-outside-toplevel
+            NetworkBuffer,
+        )
+
+        network_buffer = NetworkBuffer.get_instance()
+        created = network_buffer.fill_buffer()
+        logger_no_user_data.info(f'Initialized network buffer (created {created} networks)')
+        return
+
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
 

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -11,10 +11,10 @@ import zipfile
 from copy import copy
 from datetime import datetime
 
-import docker  # type: ignore
-import docker.types  # type: ignore
-from docker.errors import APIError, ImageNotFound  # type: ignore
-from docker.models.containers import Container  # type: ignore
+import docker
+import docker.types
+from docker.errors import APIError, ImageNotFound
+from docker.models.containers import Container
 
 from biolib import utils
 from biolib._internal.runtime import RuntimeJobDataDict
@@ -40,7 +40,7 @@ class DockerExecutor:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
-        self._absolute_image_uri = f"{utils.BIOLIB_SITE_HOSTNAME}/{self._options['module']['image_uri']}"
+        self._absolute_image_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{self._options["module"]["image_uri"]}'
         self._send_system_exception = options['send_system_exception']
         self._send_stdout_and_stderr = options['send_stdout_and_stderr']
         self._random_docker_id = compute_node_utils.random_string(15)
@@ -308,7 +308,8 @@ class DockerExecutor:
             job_uuid = self._options['job']['public_id']
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container...')
             module = self._options['module']
-            logger.debug(f"Initializing docker container with command: {module['command']}")
+            logger.debug(f'Initializing docker container with command: {module["command"]}')
+            docker_client = BiolibDockerClient.get_docker_client()
 
             docker_volume_mounts = [lfs.docker_mount for lfs in self._options['large_file_systems'].values()]
 
@@ -318,10 +319,12 @@ class DockerExecutor:
             biolib_system_secret = RuntimeJobDataDict(
                 version='1.0.0',
                 job_requested_machine=self._options['job']['requested_machine'],
+                job_requested_machine_spot=self._options['job'].get('requested_machine_spot', False),
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
                 is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
+                job_reserved_machines=self._options['job']['reserved_machines'],
             )
             docker_volume_mounts.append(
                 self._create_secrets_mount(
@@ -365,9 +368,11 @@ class DockerExecutor:
                 )
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Getting IPs for proxies...')
+
+            networks_to_connect = []
             for proxy in self._options['remote_host_proxies']:
-                proxy_ip = proxy.get_ip_address_on_network(internal_network)
                 if proxy.is_app_caller_proxy:
+                    proxy_ip = proxy.get_ip_address_on_network(internal_network)
                     logger_no_user_data.debug('Found app caller proxy, setting both base URLs in compute container')
                     environment_vars.update(
                         {
@@ -381,7 +386,11 @@ class DockerExecutor:
                         }
                     )
                 else:
-                    extra_hosts[proxy.hostname] = proxy_ip
+                    extra_hosts.update(proxy.get_hostname_to_ip_mapping())
+
+                for network in proxy.get_remote_host_networks():
+                    if network != internal_network:
+                        networks_to_connect.append(network)
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Constructing container args...')
             create_container_args = {
@@ -391,6 +400,9 @@ class DockerExecutor:
                 'mounts': docker_volume_mounts,
                 'network': internal_network.name,
                 'working_dir': module['working_directory'],
+                'networking_config': {
+                    internal_network.name: docker_client.api.create_endpoint_config(aliases=['main'])
+                },
             }
 
             if self._options['job'].get('arguments_override_command'):
@@ -429,9 +441,19 @@ class DockerExecutor:
             if docker_runtime is not None:
                 create_container_args['runtime'] = docker_runtime
 
-            docker_client = BiolibDockerClient.get_docker_client()
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Creating container...')
             self._docker_container = docker_client.containers.create(**create_container_args)
+
+            if networks_to_connect:
+                network_connection_start = time.time()
+                for network in networks_to_connect:
+                    network.connect(self._docker_container.id)
+                    logger_no_user_data.debug(f'Connected app container to network {network.name}')
+                network_connection_time = time.time() - network_connection_start
+                logger_no_user_data.debug(
+                    f'Connected app container to {len(networks_to_connect)} networks in {network_connection_time:.2f}s'
+                )
+
             logger_no_user_data.debug(f'Job "{job_uuid}" finished initializing Docker container.')
         except Exception as exception:
             raise ComputeProcessException(

biolib/compute_node/job_worker/executors/docker_types.py

@@ -1,6 +1,6 @@
 import enum
 
-from biolib.typing_utils import TypedDict, Any
+from biolib.typing_utils import Any, TypedDict
 
 
 class Proxy(TypedDict):

biolib/compute_node/job_worker/executors/types.py

@@ -1,11 +1,11 @@
-from docker.models.networks import Network  # type: ignore
+from docker.models.networks import Network
 
+from biolib.biolib_api_client.app_types import Module
+from biolib.biolib_api_client.job_types import CloudJob, CreatedJobDict
 from biolib.compute_node.job_worker.large_file_system import LargeFileSystem
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Callable, Optional, List, Dict
 from biolib.compute_node.remote_host_proxy import RemoteHostProxy
-from biolib.biolib_api_client.app_types import Module
-from biolib.biolib_api_client.job_types import CreatedJobDict, CloudJob
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Callable, Dict, List, Optional, TypedDict
 
 
 class StatusUpdate(TypedDict):
@@ -43,6 +43,7 @@ class LocalExecutorOptions(TypedDict):
     send_system_exception: SendSystemExceptionType
     send_stdout_and_stderr: SendStdoutAndStderrType
 
+
 class MetadataToSaveOutput(TypedDict):
     arguments: List[str]
     startup_error_string: Optional[str]