pybiolib 1.2.1056__py3-none-any.whl → 1.2.1642__py3-none-any.whl

biolib/biolib_api_client/app_types.py

@@ -68,11 +68,6 @@ class LargeFileSystemMapping(TypedDict):
     uuid: str
 
 
-class PortMapping(TypedDict):
-    from_port: int
-    to_port: int
-
-
 class _Module(TypedDict):
     command: str
     environment: Literal['biolib-app', 'biolib-custom', 'biolib-ecr']
@@ -83,7 +78,7 @@ class _Module(TypedDict):
     large_file_systems: List[LargeFileSystemMapping]
     name: str
     output_files_mappings: List[FilesMapping]
-    port_mappings: List[PortMapping]
+    ports: List[int]
     source_files_mappings: List[FilesMapping]
     working_directory: str
 

biolib/biolib_api_client/biolib_app_api.py

@@ -57,6 +57,22 @@ def _get_git_branch_name() -> str:
         return ''
 
 
+def _get_git_commit_hash() -> str:
+    try:
+        github_actions_commit_hash = os.getenv('GITHUB_SHA')
+        if github_actions_commit_hash:
+            return github_actions_commit_hash
+
+        gitlab_ci_commit_hash = os.getenv('CI_COMMIT_SHA')
+        if gitlab_ci_commit_hash:
+            return gitlab_ci_commit_hash
+
+        result = subprocess.run(['git', 'rev-parse', 'HEAD'], check=True, stdout=subprocess.PIPE, text=True)
+        return result.stdout.strip()
+    except BaseException:
+        return ''
+
+
 def _get_git_repository_url() -> str:
     try:
         result = subprocess.run(['git', 'remote', 'get-url', 'origin'], check=True, stdout=subprocess.PIPE, text=True)
@@ -125,6 +141,7 @@ class BiolibAppApi:
                 'state': 'published',
                 'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
                 'git_branch_name': _get_git_branch_name(),
+                'git_commit_hash': _get_git_commit_hash(),
                 'git_repository_url': _get_git_repository_url(),
             }
             if semantic_version:

biolib/biolib_binary_format/module_input.py

@@ -1,4 +1,5 @@
 from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
+from biolib.biolib_logging import logger
 from biolib.typing_utils import TypedDict, Dict, List
 
 
@@ -14,6 +15,10 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
         self.package_type = 1
 
     def serialize(self, stdin, arguments, files) -> bytes:
+        for path in files.keys():
+            if '//' in path:
+                raise ValueError(f"File path '{path}' contains double slashes which are not allowed")
+
         bbf_data = bytearray()
         bbf_data.extend(self.version.to_bytes(1, 'big'))
         bbf_data.extend(self.package_type.to_bytes(1, 'big'))
@@ -67,6 +72,9 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
             data_len = self.get_data(8, output_type='int')
             path = self.get_data(path_len, output_type='str')
             data = self.get_data(data_len)
+            if '//' in path:
+                # TODO: Raise ValueError here once backwards compatibility period is over
+                logger.warning(f"File path '{path}' contains double slashes which are not allowed")
             files[path] = bytes(data)
 
         return ModuleInputDict(stdin=stdin, arguments=arguments, files=files)

biolib/biolib_binary_format/remote_endpoints.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format.utils import RemoteEndpoint
@@ -17,13 +17,13 @@ class RemoteJobStorageEndpoint(RemoteEndpoint):
         self._storage_type: Literal['input', 'output'] = storage_type
 
     def get_remote_url(self):
-        if not self._presigned_url or datetime.utcnow() > self._expires_at:
+        if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
             self._presigned_url = BiolibJobApi.get_job_storage_download_url(
                 job_auth_token=self._job_auth_token,
                 job_uuid=self._job_uuid,
                 storage_type='results' if self._storage_type == 'output' else 'input',
             )
-            self._expires_at = datetime.utcnow() + timedelta(minutes=8)
+            self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
             # TODO: Use expires at from url
             # parsed_url = urlparse(self._presigned_url)
             # query_params = parse_qs(parsed_url.query)

biolib/biolib_binary_format/remote_stream_seeker.py

@@ -1,45 +1,59 @@
 from biolib.biolib_binary_format.utils import IndexableBuffer
+from biolib.biolib_logging import logger
 from biolib.typing_utils import Iterable
 
 
 class StreamSeeker:
     def __init__(
-            self,
-            upstream_buffer: IndexableBuffer,
-            files_data_start: int,
-            files_data_end: int,
-            download_chunk_size_in_bytes: int,
+        self,
+        upstream_buffer: IndexableBuffer,
+        files_data_start: int,
+        files_data_end: int,
+        max_chunk_size: int,
     ):
         self._upstream_buffer = upstream_buffer
         self._files_data_end = files_data_end
-        self._download_chunk_size_in_bytes = download_chunk_size_in_bytes
+        self._max_chunk_size = max_chunk_size
 
         self._buffer_start = files_data_start
         self._buffer = bytearray()
 
-    def seek_and_read(self, file_start: int, file_length: int) -> Iterable[bytes]:
+    def seek_and_read(self, file_start: int, file_length: int, read_ahead_bytes: int = 0) -> Iterable[bytes]:
         assert file_start >= self._buffer_start
-        self._buffer = self._buffer[file_start - self._buffer_start:]  # Returns empty array if "out of bounds"
+        self._buffer = self._buffer[file_start - self._buffer_start :]
         self._buffer_start = file_start
 
         while True:
             file_byte_count_remaining = file_length - (self._buffer_start - file_start)
-            if file_byte_count_remaining == 0:
+            if file_byte_count_remaining <= 0:
                 return
 
-            start_of_fetch = self._buffer_start + len(self._buffer)
-            byte_count_left_in_stream = self._files_data_end - start_of_fetch
-
-            if byte_count_left_in_stream != 0:
-                # Only fetch if there is still data left upstream
-                if self._download_chunk_size_in_bytes > len(self._buffer):
-                    # Only fetch if size of buffer is below chunk size
-                    self._buffer.extend(self._upstream_buffer.get_data(
-                        start=start_of_fetch,
-                        length=min(byte_count_left_in_stream, self._download_chunk_size_in_bytes),
-                    ))
-
-            bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
-            yield bytes_to_yield
-            self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
-            self._buffer_start += len(bytes_to_yield)
+            if len(self._buffer) > 0:
+                take = min(file_byte_count_remaining, len(self._buffer))
+                chunk = self._buffer[:take]
+                if chunk:
+                    yield chunk
+                self._buffer = self._buffer[take:]
+                self._buffer_start += take
+            else:
+                start_of_fetch = self._buffer_start + len(self._buffer)
+                bytes_left_in_stream = self._files_data_end - start_of_fetch
+                if bytes_left_in_stream <= 0:
+                    logger.error(
+                        'StreamSeeker: no bytes left upstream (start_of_fetch=%d, files_data_end=%d)',
+                        start_of_fetch,
+                        self._files_data_end,
+                    )
+                    return
+
+                fetch_size = min(self._max_chunk_size, file_byte_count_remaining + read_ahead_bytes)
+                if fetch_size > bytes_left_in_stream:
+                    logger.error(
+                        'StreamSeeker: fetch_size (%d) > bytes_left_in_stream (%d); clamping',
+                        fetch_size,
+                        bytes_left_in_stream,
+                    )
+                    fetch_size = bytes_left_in_stream
+
+                fetched_data = self._upstream_buffer.get_data(start=start_of_fetch, length=fetch_size)
+                self._buffer.extend(fetched_data)

biolib/cli/__init__.py

@@ -5,7 +5,7 @@ import click
 
 from biolib import utils
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.cli import auth, data_record, download_container, init, lfs, push, run, runtime, sdk, start
+from biolib.cli import auth, data_record, download_container, index, init, lfs, push, run, runtime, sdk, start
 
 
 @click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
@@ -31,6 +31,7 @@ cli.add_command(run.run)
 cli.add_command(runtime.runtime)
 cli.add_command(start.start)
 cli.add_command(data_record.data_record)
+cli.add_command(index.index)
 cli.add_command(sdk.sdk)
 
 # allow this script to be called without poetry in dev e.g. by an IDE debugger

biolib/cli/data_record.py

@@ -80,3 +80,20 @@ def describe(uri: str, output_as_json: bool) -> None:
             size_string = str(file_info['size_bytes'])
             leading_space_string = ' ' * (10 - len(size_string))
             print(f"{leading_space_string}{size_string}    {file_info['path']}")
+
+
+@data_record.command(help='Delete a Data Record')
+@click.argument('uri', required=True)
+def delete(uri: str) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+
+    print(f'You are about to delete the data record: {record.uri}')
+    print('This action cannot be undone.')
+
+    confirmation = input(f'To confirm deletion, please type the data record name "{record.name}": ')
+    if confirmation != record.name:
+        print('Data record name does not match. Deletion cancelled.')
+        return
+
+    record.delete()
+    print(f'Data record {record.uri} has been deleted.')

biolib/cli/index.py

@@ -0,0 +1,32 @@
+import json
+import logging
+import sys
+
+import click
+
+from biolib._index.index import Index
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger, logger_no_user_data
+
+
+@click.group(help='Manage Indexes')
+def index() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@index.command(help='Create an Index')
+@click.argument('uri', required=True)
+@click.option('--config-path', required=True, type=click.Path(exists=True), help='Path to JSON config file')
+def create(uri: str, config_path: str) -> None:
+    try:
+        Index.create_from_config_file(uri=uri, config_path=config_path)
+    except json.JSONDecodeError as error:
+        print(f'Error: Invalid JSON in config file: {error}', file=sys.stderr)
+        sys.exit(1)
+    except BioLibError as error:
+        print(f'Error creating index: {error.message}', file=sys.stderr)
+        sys.exit(1)
+    except Exception as error:
+        print(f'Error reading config file: {error}', file=sys.stderr)
+        sys.exit(1)

biolib/cli/lfs.py

@@ -13,7 +13,7 @@ from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import Optional
 
 
-@click.group(help='Manage Large File Systems')
+@click.group(help='Manage Large File Systems', hidden=True)
 def lfs() -> None:
     pass
 

biolib/cli/start.py

@@ -13,7 +13,10 @@ from biolib.typing_utils import Optional
 @click.option('--port', default=5000, type=click.IntRange(1, 65_535), required=False)
 @click.option('--tls-certificate', type=click.Path(exists=True), required=False, hidden=True)
 @click.option('--tls-key', type=click.Path(exists=True), required=False, hidden=True)
-def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str]) -> None:
+@click.option('--initialize-network-buffer', is_flag=True, help='Initialize the remote host network buffer and exit')
+def start(
+    host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str], initialize_network_buffer: bool
+) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     if platform.system() == 'Windows':
@@ -22,6 +25,16 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
     if tls_certificate and not tls_key or tls_key and not tls_certificate:
         raise Exception('Options --tls-certificate and --tls-key must be specified together')
 
+    if initialize_network_buffer:
+        from biolib.compute_node.job_worker.network_buffer import (  # pylint: disable=import-outside-toplevel
+            NetworkBuffer,
+        )
+
+        network_buffer = NetworkBuffer.get_instance()
+        created = network_buffer.fill_buffer()
+        logger_no_user_data.info(f'Initialized network buffer (created {created} networks)')
+        return
+
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
 

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -11,10 +11,10 @@ import zipfile
 from copy import copy
 from datetime import datetime
 
-import docker  # type: ignore
-import docker.types  # type: ignore
-from docker.errors import APIError, ImageNotFound  # type: ignore
-from docker.models.containers import Container  # type: ignore
+import docker
+import docker.types
+from docker.errors import APIError, ImageNotFound
+from docker.models.containers import Container
 
 from biolib import utils
 from biolib._internal.runtime import RuntimeJobDataDict
@@ -40,7 +40,7 @@ class DockerExecutor:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
-        self._absolute_image_uri = f"{utils.BIOLIB_SITE_HOSTNAME}/{self._options['module']['image_uri']}"
+        self._absolute_image_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{self._options["module"]["image_uri"]}'
         self._send_system_exception = options['send_system_exception']
         self._send_stdout_and_stderr = options['send_stdout_and_stderr']
         self._random_docker_id = compute_node_utils.random_string(15)
@@ -308,7 +308,8 @@ class DockerExecutor:
             job_uuid = self._options['job']['public_id']
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container...')
             module = self._options['module']
-            logger.debug(f"Initializing docker container with command: {module['command']}")
+            logger.debug(f'Initializing docker container with command: {module["command"]}')
+            docker_client = BiolibDockerClient.get_docker_client()
 
             docker_volume_mounts = [lfs.docker_mount for lfs in self._options['large_file_systems'].values()]
 
@@ -318,10 +319,12 @@ class DockerExecutor:
             biolib_system_secret = RuntimeJobDataDict(
                 version='1.0.0',
                 job_requested_machine=self._options['job']['requested_machine'],
+                job_requested_machine_spot=self._options['job'].get('requested_machine_spot', False),
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
                 is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
+                job_reserved_machines=self._options['job']['reserved_machines'],
             )
             docker_volume_mounts.append(
                 self._create_secrets_mount(
@@ -365,9 +368,11 @@ class DockerExecutor:
                 )
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Getting IPs for proxies...')
+
+            networks_to_connect = []
             for proxy in self._options['remote_host_proxies']:
-                proxy_ip = proxy.get_ip_address_on_network(internal_network)
                 if proxy.is_app_caller_proxy:
+                    proxy_ip = proxy.get_ip_address_on_network(internal_network)
                     logger_no_user_data.debug('Found app caller proxy, setting both base URLs in compute container')
                     environment_vars.update(
                         {
@@ -381,7 +386,11 @@ class DockerExecutor:
                         }
                     )
                 else:
-                    extra_hosts[proxy.hostname] = proxy_ip
+                    extra_hosts.update(proxy.get_hostname_to_ip_mapping())
+
+                for network in proxy.get_remote_host_networks():
+                    if network != internal_network:
+                        networks_to_connect.append(network)
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Constructing container args...')
             create_container_args = {
@@ -391,6 +400,9 @@ class DockerExecutor:
                 'mounts': docker_volume_mounts,
                 'network': internal_network.name,
                 'working_dir': module['working_directory'],
+                'networking_config': {
+                    internal_network.name: docker_client.api.create_endpoint_config(aliases=['main'])
+                },
             }
 
             if self._options['job'].get('arguments_override_command'):
@@ -429,9 +441,19 @@ class DockerExecutor:
             if docker_runtime is not None:
                 create_container_args['runtime'] = docker_runtime
 
-            docker_client = BiolibDockerClient.get_docker_client()
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Creating container...')
             self._docker_container = docker_client.containers.create(**create_container_args)
+
+            if networks_to_connect:
+                network_connection_start = time.time()
+                for network in networks_to_connect:
+                    network.connect(self._docker_container.id)
+                    logger_no_user_data.debug(f'Connected app container to network {network.name}')
+                network_connection_time = time.time() - network_connection_start
+                logger_no_user_data.debug(
+                    f'Connected app container to {len(networks_to_connect)} networks in {network_connection_time:.2f}s'
+                )
+
             logger_no_user_data.debug(f'Job "{job_uuid}" finished initializing Docker container.')
         except Exception as exception:
             raise ComputeProcessException(

biolib/compute_node/job_worker/executors/docker_types.py

@@ -1,6 +1,6 @@
 import enum
 
-from biolib.typing_utils import TypedDict, Any
+from biolib.typing_utils import Any, TypedDict
 
 
 class Proxy(TypedDict):

biolib/compute_node/job_worker/executors/types.py

@@ -1,11 +1,11 @@
-from docker.models.networks import Network  # type: ignore
+from docker.models.networks import Network
 
+from biolib.biolib_api_client.app_types import Module
+from biolib.biolib_api_client.job_types import CloudJob, CreatedJobDict
 from biolib.compute_node.job_worker.large_file_system import LargeFileSystem
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Callable, Optional, List, Dict
 from biolib.compute_node.remote_host_proxy import RemoteHostProxy
-from biolib.biolib_api_client.app_types import Module
-from biolib.biolib_api_client.job_types import CreatedJobDict, CloudJob
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Callable, Dict, List, Optional, TypedDict
 
 
 class StatusUpdate(TypedDict):
@@ -43,6 +43,7 @@ class LocalExecutorOptions(TypedDict):
     send_system_exception: SendSystemExceptionType
     send_stdout_and_stderr: SendStdoutAndStderrType
 
+
 class MetadataToSaveOutput(TypedDict):
     arguments: List[str]
     startup_error_string: Optional[str]