pybiolib 1.2.1056__py3-none-any.whl → 1.2.1727__py3-none-any.whl

biolib/biolib_binary_format/module_input.py

@@ -1,4 +1,5 @@
 from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
+from biolib.biolib_logging import logger
 from biolib.typing_utils import TypedDict, Dict, List
 
 
@@ -14,6 +15,10 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
         self.package_type = 1
 
     def serialize(self, stdin, arguments, files) -> bytes:
+        for path in files.keys():
+            if '//' in path:
+                raise ValueError(f"File path '{path}' contains double slashes which are not allowed")
+
         bbf_data = bytearray()
         bbf_data.extend(self.version.to_bytes(1, 'big'))
         bbf_data.extend(self.package_type.to_bytes(1, 'big'))
@@ -67,6 +72,9 @@ class ModuleInput(BioLibBinaryFormatBasePackage):
             data_len = self.get_data(8, output_type='int')
             path = self.get_data(path_len, output_type='str')
             data = self.get_data(data_len)
+            if '//' in path:
+                # TODO: Raise ValueError here once backwards compatibility period is over
+                logger.warning(f"File path '{path}' contains double slashes which are not allowed")
             files[path] = bytes(data)
 
         return ModuleInputDict(stdin=stdin, arguments=arguments, files=files)

biolib/biolib_binary_format/remote_endpoints.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format.utils import RemoteEndpoint
@@ -17,13 +17,13 @@ class RemoteJobStorageEndpoint(RemoteEndpoint):
         self._storage_type: Literal['input', 'output'] = storage_type
 
     def get_remote_url(self):
-        if not self._presigned_url or datetime.utcnow() > self._expires_at:
+        if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
             self._presigned_url = BiolibJobApi.get_job_storage_download_url(
                 job_auth_token=self._job_auth_token,
                 job_uuid=self._job_uuid,
                 storage_type='results' if self._storage_type == 'output' else 'input',
             )
-            self._expires_at = datetime.utcnow() + timedelta(minutes=8)
+            self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
             # TODO: Use expires at from url
             # parsed_url = urlparse(self._presigned_url)
             # query_params = parse_qs(parsed_url.query)

biolib/biolib_binary_format/remote_stream_seeker.py

@@ -1,45 +1,59 @@
 from biolib.biolib_binary_format.utils import IndexableBuffer
+from biolib.biolib_logging import logger
 from biolib.typing_utils import Iterable
 
 
 class StreamSeeker:
     def __init__(
-            self,
-            upstream_buffer: IndexableBuffer,
-            files_data_start: int,
-            files_data_end: int,
-            download_chunk_size_in_bytes: int,
+        self,
+        upstream_buffer: IndexableBuffer,
+        files_data_start: int,
+        files_data_end: int,
+        max_chunk_size: int,
     ):
         self._upstream_buffer = upstream_buffer
         self._files_data_end = files_data_end
-        self._download_chunk_size_in_bytes = download_chunk_size_in_bytes
+        self._max_chunk_size = max_chunk_size
 
         self._buffer_start = files_data_start
         self._buffer = bytearray()
 
-    def seek_and_read(self, file_start: int, file_length: int) -> Iterable[bytes]:
+    def seek_and_read(self, file_start: int, file_length: int, read_ahead_bytes: int = 0) -> Iterable[bytes]:
         assert file_start >= self._buffer_start
-        self._buffer = self._buffer[file_start - self._buffer_start:]  # Returns empty array if "out of bounds"
+        self._buffer = self._buffer[file_start - self._buffer_start :]
         self._buffer_start = file_start
 
         while True:
             file_byte_count_remaining = file_length - (self._buffer_start - file_start)
-            if file_byte_count_remaining == 0:
+            if file_byte_count_remaining <= 0:
                 return
 
-            start_of_fetch = self._buffer_start + len(self._buffer)
-            byte_count_left_in_stream = self._files_data_end - start_of_fetch
-
-            if byte_count_left_in_stream != 0:
-                # Only fetch if there is still data left upstream
-                if self._download_chunk_size_in_bytes > len(self._buffer):
-                    # Only fetch if size of buffer is below chunk size
-                    self._buffer.extend(self._upstream_buffer.get_data(
-                        start=start_of_fetch,
-                        length=min(byte_count_left_in_stream, self._download_chunk_size_in_bytes),
-                    ))
-
-            bytes_to_yield = self._buffer[:file_byte_count_remaining]  # Returns empty array if "out of bounds"
-            yield bytes_to_yield
-            self._buffer = self._buffer[file_byte_count_remaining:]  # Returns empty array if "out of bounds"
-            self._buffer_start += len(bytes_to_yield)
+            if len(self._buffer) > 0:
+                take = min(file_byte_count_remaining, len(self._buffer))
+                chunk = self._buffer[:take]
+                if chunk:
+                    yield chunk
+                self._buffer = self._buffer[take:]
+                self._buffer_start += take
+            else:
+                start_of_fetch = self._buffer_start + len(self._buffer)
+                bytes_left_in_stream = self._files_data_end - start_of_fetch
+                if bytes_left_in_stream <= 0:
+                    logger.error(
+                        'StreamSeeker: no bytes left upstream (start_of_fetch=%d, files_data_end=%d)',
+                        start_of_fetch,
+                        self._files_data_end,
+                    )
+                    return
+
+                fetch_size = min(self._max_chunk_size, file_byte_count_remaining + read_ahead_bytes)
+                if fetch_size > bytes_left_in_stream:
+                    logger.error(
+                        'StreamSeeker: fetch_size (%d) > bytes_left_in_stream (%d); clamping',
+                        fetch_size,
+                        bytes_left_in_stream,
+                    )
+                    fetch_size = bytes_left_in_stream
+
+                fetched_data = self._upstream_buffer.get_data(start=start_of_fetch, length=fetch_size)
+                self._buffer.extend(fetched_data)

biolib/cli/__init__.py

@@ -5,7 +5,7 @@ import click
 
 from biolib import utils
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.cli import auth, data_record, download_container, init, lfs, push, run, runtime, sdk, start
+from biolib.cli import auth, data_record, download_container, index, init, lfs, push, run, runtime, sdk, start
 
 
 @click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
@@ -31,6 +31,7 @@ cli.add_command(run.run)
 cli.add_command(runtime.runtime)
 cli.add_command(start.start)
 cli.add_command(data_record.data_record)
+cli.add_command(index.index)
 cli.add_command(sdk.sdk)
 
 # allow this script to be called without poetry in dev e.g. by an IDE debugger

biolib/cli/data_record.py

@@ -1,9 +1,11 @@
 import json
 import logging
 import os
+import sys
 from typing import Dict, List
 
 import click
+import rich.progress
 
 from biolib._data_record.data_record import DataRecord
 from biolib.biolib_api_client import BiolibApiClient
@@ -80,3 +82,83 @@ def describe(uri: str, output_as_json: bool) -> None:
             size_string = str(file_info['size_bytes'])
             leading_space_string = ' ' * (10 - len(size_string))
             print(f"{leading_space_string}{size_string}    {file_info['path']}")
+
+
+@data_record.command(help='Delete a Data Record')
+@click.argument('uri', required=True)
+def delete(uri: str) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+
+    print(f'You are about to delete the data record: {record.uri}')
+    print('This action cannot be undone.')
+
+    confirmation = input(f'To confirm deletion, please type the data record name "{record.name}": ')
+    if confirmation != record.name:
+        print('Data record name does not match. Deletion cancelled.')
+        return
+
+    record.delete()
+    print(f'Data record {record.uri} has been deleted.')
+
+
+def _clone_data_record_with_progress(
+    source_record: DataRecord,
+    dest_record: DataRecord,
+) -> None:
+    # pylint: disable=protected-access
+    total_size_in_bytes = source_record._get_zip_size_bytes()
+    # pylint: enable=protected-access
+
+    if total_size_in_bytes == 0:
+        logger.info('Source data record has no data to clone')
+        return
+
+    if sys.stdout.isatty():
+        with rich.progress.Progress(
+            rich.progress.TextColumn('[bold blue]{task.description}'),
+            rich.progress.BarColumn(),
+            rich.progress.TaskProgressColumn(),
+            rich.progress.TimeRemainingColumn(),
+            rich.progress.TransferSpeedColumn(),
+        ) as progress:
+            task_id = progress.add_task('Cloning data record', total=total_size_in_bytes)
+
+            def on_progress(bytes_uploaded: int, _total_bytes: int) -> None:
+                progress.update(task_id, completed=bytes_uploaded)
+
+            DataRecord.clone(source=source_record, destination=dest_record, on_progress=on_progress)
+    else:
+        logger.info(f'Cloning ~{round(total_size_in_bytes / 10**6)}mb of data')
+        DataRecord.clone(source=source_record, destination=dest_record)
+
+
+def _get_or_create_destination_record(destination_uri: str) -> Optional[DataRecord]:
+    try:
+        return DataRecord.get_by_uri(uri=destination_uri)
+    except Exception:
+        print(f'Destination data record "{destination_uri}" does not exist.')
+        confirmation = input('Would you like to create it? [y/N]: ')
+        if confirmation.lower() != 'y':
+            print('Clone cancelled.')
+            return None
+
+        return DataRecord.create(destination=destination_uri)
+
+
+@data_record.command(help='Clone a Data Record to another location')
+@click.argument('source_uri', required=True)
+@click.argument('destination_uri', required=True)
+def clone(source_uri: str, destination_uri: str) -> None:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='clone a Data Record')
+
+    logger.info(f'Fetching source data record: {source_uri}')
+    source_record = DataRecord.get_by_uri(uri=source_uri)
+
+    logger.info(f'Checking destination data record: {destination_uri}')
+    dest_record = _get_or_create_destination_record(destination_uri)
+    if dest_record is None:
+        return
+
+    logger.info(f'Cloning from {source_record.uri} to {dest_record.uri}...')
+    _clone_data_record_with_progress(source_record=source_record, dest_record=dest_record)
+    logger.info('Clone completed successfully.')

biolib/cli/index.py

@@ -0,0 +1,32 @@
+import json
+import logging
+import sys
+
+import click
+
+from biolib._index.index import Index
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger, logger_no_user_data
+
+
+@click.group(help='Manage Indexes')
+def index() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@index.command(help='Create an Index')
+@click.argument('uri', required=True)
+@click.option('--config-path', required=True, type=click.Path(exists=True), help='Path to JSON config file')
+def create(uri: str, config_path: str) -> None:
+    try:
+        Index.create_from_config_file(uri=uri, config_path=config_path)
+    except json.JSONDecodeError as error:
+        print(f'Error: Invalid JSON in config file: {error}', file=sys.stderr)
+        sys.exit(1)
+    except BioLibError as error:
+        print(f'Error creating index: {error.message}', file=sys.stderr)
+        sys.exit(1)
+    except Exception as error:
+        print(f'Error reading config file: {error}', file=sys.stderr)
+        sys.exit(1)

biolib/cli/init.py

@@ -10,18 +10,56 @@ from biolib import (
 )
 from biolib._internal.add_copilot_prompts import add_copilot_prompts
 from biolib._internal.add_gui_files import add_gui_files
-from biolib._internal.http_client import HttpError
+from biolib._internal.http_client import HttpClient, HttpError
 from biolib._internal.string_utils import normalize_for_docker_tag
 from biolib._internal.templates import templates
 from biolib.api import client as api_client
 from biolib.biolib_api_client.api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
+from biolib.biolib_logging import logger_no_user_data
+from biolib.typing_utils import Optional
 from biolib.user.sign_in import sign_in
 from biolib.utils import BIOLIB_PACKAGE_VERSION
 
 
+def _get_latest_pypi_version() -> Optional[str]:
+    try:
+        response = HttpClient.request(
+            url='https://pypi.org/pypi/pybiolib/json',
+            timeout_in_seconds=5,
+            retries=1,
+        )
+        data = response.json()
+        version = data.get('info', {}).get('version')
+        if isinstance(version, str):
+            return version
+        return None
+    except Exception as error:
+        logger_no_user_data.debug(f'Failed to fetch latest version from PyPI: {error}')
+        return None
+
+
+def _is_current_version_outdated(current: str, latest: str) -> bool:
+    try:
+        current_parts = [int(x) for x in current.split('.')]
+        latest_parts = [int(x) for x in latest.split('.')]
+        return current_parts < latest_parts
+    except (ValueError, AttributeError):
+        return False
+
+
 @click.command(help='Initialize a BioLib project', hidden=True)
 def init() -> None:
+    latest_version = _get_latest_pypi_version()
+    if latest_version and _is_current_version_outdated(BIOLIB_PACKAGE_VERSION, latest_version):
+        print(f'A newer version of pybiolib is available: {latest_version} (current: {BIOLIB_PACKAGE_VERSION})')
+        print('To upgrade, run: pip install --upgrade pybiolib')
+        print()
+        continue_input = input('Do you want to continue with the current version? [y/N]: ')
+        if continue_input.lower() not in ['y', 'yes']:
+            print('Please upgrade pybiolib and run `biolib init` again.')
+            return
+
     cwd = os.getcwd()
 
     app_uri = input('What URI do you want to create the application under? (leave blank to skip): ')

biolib/cli/lfs.py

@@ -13,7 +13,7 @@ from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import Optional
 
 
-@click.group(help='Manage Large File Systems')
+@click.group(help='Manage Large File Systems', hidden=True)
 def lfs() -> None:
     pass
 

biolib/cli/run.py

@@ -13,19 +13,23 @@ from biolib.typing_utils import Optional, Tuple
     help='Run an application on BioLib.',
 )
 @click.option('--experiment', type=str, required=False, help='Experiment name or URI to add the run to.')
-@click.option('--local', is_flag=True, required=False, help='Run the application locally.')
+@click.option('--local', is_flag=True, required=False, hidden=True)
 @click.option('--non-blocking', is_flag=True, required=False, help='Run the application non blocking.')
 @click.argument('uri', required=True)
 @click.argument('args', nargs=-1, type=click.UNPROCESSED)
 def run(experiment: Optional[str], local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
+    if local:
+        print('Error: Running applications locally with --local is no longer supported.', file=sys.stderr)
+        sys.exit(1)
+
     if experiment:
         with Experiment(uri=experiment):
-            _run(local=local, non_blocking=non_blocking, uri=uri, args=args)
+            _run(non_blocking=non_blocking, uri=uri, args=args)
     else:
-        _run(local=local, non_blocking=non_blocking, uri=uri, args=args)
+        _run(non_blocking=non_blocking, uri=uri, args=args)
 
 
-def _run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
+def _run(non_blocking: bool, uri: str, args: Tuple[str]) -> None:
     try:
         app = BioLibApp(uri=uri)
     except biolib_errors.BioLibError as error:
@@ -43,7 +47,6 @@ def _run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
         args=list(args),
         stdin=_get_stdin(),
         files=None,
-        machine=('local' if local else ''),
         blocking=blocking,
     )
 

biolib/cli/start.py

@@ -13,7 +13,10 @@ from biolib.typing_utils import Optional
 @click.option('--port', default=5000, type=click.IntRange(1, 65_535), required=False)
 @click.option('--tls-certificate', type=click.Path(exists=True), required=False, hidden=True)
 @click.option('--tls-key', type=click.Path(exists=True), required=False, hidden=True)
-def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str]) -> None:
+@click.option('--initialize-network-buffer', is_flag=True, help='Initialize the remote host network buffer and exit')
+def start(
+    host: str, port: int, tls_certificate: Optional[str], tls_key: Optional[str], initialize_network_buffer: bool
+) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     if platform.system() == 'Windows':
@@ -22,6 +25,16 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
     if tls_certificate and not tls_key or tls_key and not tls_certificate:
         raise Exception('Options --tls-certificate and --tls-key must be specified together')
 
+    if initialize_network_buffer:
+        from biolib.compute_node.job_worker.network_buffer import (  # pylint: disable=import-outside-toplevel
+            NetworkBuffer,
+        )
+
+        network_buffer = NetworkBuffer.get_instance()
+        created = network_buffer.fill_buffer()
+        logger_no_user_data.info(f'Initialized network buffer (created {created} networks)')
+        return
+
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
 

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -11,10 +11,10 @@ import zipfile
 from copy import copy
 from datetime import datetime
 
-import docker  # type: ignore
-import docker.types  # type: ignore
-from docker.errors import APIError, ImageNotFound  # type: ignore
-from docker.models.containers import Container  # type: ignore
+import docker
+import docker.types
+from docker.errors import APIError, ImageNotFound
+from docker.models.containers import Container
 
 from biolib import utils
 from biolib._internal.runtime import RuntimeJobDataDict
@@ -40,7 +40,7 @@ class DockerExecutor:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
-        self._absolute_image_uri = f"{utils.BIOLIB_SITE_HOSTNAME}/{self._options['module']['image_uri']}"
+        self._absolute_image_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{self._options["module"]["image_uri"]}'
         self._send_system_exception = options['send_system_exception']
         self._send_stdout_and_stderr = options['send_stdout_and_stderr']
         self._random_docker_id = compute_node_utils.random_string(15)
@@ -308,7 +308,8 @@ class DockerExecutor:
             job_uuid = self._options['job']['public_id']
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container...')
             module = self._options['module']
-            logger.debug(f"Initializing docker container with command: {module['command']}")
+            logger.debug(f'Initializing docker container with command: {module["command"]}')
+            docker_client = BiolibDockerClient.get_docker_client()
 
             docker_volume_mounts = [lfs.docker_mount for lfs in self._options['large_file_systems'].values()]
 
@@ -318,10 +319,12 @@ class DockerExecutor:
             biolib_system_secret = RuntimeJobDataDict(
                 version='1.0.0',
                 job_requested_machine=self._options['job']['requested_machine'],
+                job_requested_machine_spot=self._options['job'].get('requested_machine_spot', False),
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
                 is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
+                job_reserved_machines=self._options['job']['reserved_machines'],
             )
             docker_volume_mounts.append(
                 self._create_secrets_mount(
@@ -365,9 +368,11 @@ class DockerExecutor:
                 )
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Getting IPs for proxies...')
+
+            networks_to_connect = []
             for proxy in self._options['remote_host_proxies']:
-                proxy_ip = proxy.get_ip_address_on_network(internal_network)
                 if proxy.is_app_caller_proxy:
+                    proxy_ip = proxy.get_ip_address_on_network(internal_network)
                     logger_no_user_data.debug('Found app caller proxy, setting both base URLs in compute container')
                     environment_vars.update(
                         {
@@ -381,7 +386,11 @@ class DockerExecutor:
                         }
                     )
                 else:
-                    extra_hosts[proxy.hostname] = proxy_ip
+                    extra_hosts.update(proxy.get_hostname_to_ip_mapping())
+
+                for network in proxy.get_remote_host_networks():
+                    if network != internal_network:
+                        networks_to_connect.append(network)
 
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Constructing container args...')
             create_container_args = {
@@ -391,6 +400,9 @@ class DockerExecutor:
                 'mounts': docker_volume_mounts,
                 'network': internal_network.name,
                 'working_dir': module['working_directory'],
+                'networking_config': {
+                    internal_network.name: docker_client.api.create_endpoint_config(aliases=['main'])
+                },
             }
 
             if self._options['job'].get('arguments_override_command'):
@@ -429,9 +441,19 @@ class DockerExecutor:
             if docker_runtime is not None:
                 create_container_args['runtime'] = docker_runtime
 
-            docker_client = BiolibDockerClient.get_docker_client()
             logger_no_user_data.debug(f'Job "{job_uuid}" initializing Docker container. Creating container...')
             self._docker_container = docker_client.containers.create(**create_container_args)
+
+            if networks_to_connect:
+                network_connection_start = time.time()
+                for network in networks_to_connect:
+                    network.connect(self._docker_container.id)
+                    logger_no_user_data.debug(f'Connected app container to network {network.name}')
+                network_connection_time = time.time() - network_connection_start
+                logger_no_user_data.debug(
+                    f'Connected app container to {len(networks_to_connect)} networks in {network_connection_time:.2f}s'
+                )
+
             logger_no_user_data.debug(f'Job "{job_uuid}" finished initializing Docker container.')
         except Exception as exception:
             raise ComputeProcessException(

biolib/compute_node/job_worker/executors/docker_types.py

@@ -1,6 +1,6 @@
 import enum
 
-from biolib.typing_utils import TypedDict, Any
+from biolib.typing_utils import Any, TypedDict
 
 
 class Proxy(TypedDict):

biolib/compute_node/job_worker/executors/types.py

@@ -1,11 +1,11 @@
-from docker.models.networks import Network  # type: ignore
+from docker.models.networks import Network
 
+from biolib.biolib_api_client.app_types import Module
+from biolib.biolib_api_client.job_types import CloudJob, CreatedJobDict
 from biolib.compute_node.job_worker.large_file_system import LargeFileSystem
-from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
-from biolib.typing_utils import TypedDict, Callable, Optional, List, Dict
 from biolib.compute_node.remote_host_proxy import RemoteHostProxy
-from biolib.biolib_api_client.app_types import Module
-from biolib.biolib_api_client.job_types import CreatedJobDict, CloudJob
+from biolib.compute_node.webserver.webserver_types import ComputeNodeInfo
+from biolib.typing_utils import Callable, Dict, List, Optional, TypedDict
 
 
 class StatusUpdate(TypedDict):
@@ -43,6 +43,7 @@ class LocalExecutorOptions(TypedDict):
     send_system_exception: SendSystemExceptionType
     send_stdout_and_stderr: SendStdoutAndStderrType
 
+
 class MetadataToSaveOutput(TypedDict):
     arguments: List[str]
     startup_error_string: Optional[str]