pybiolib 1.1.2193__py3-none-any.whl → 1.2.105.dev1__py3-none-any.whl

biolib/__init__.py

@@ -41,8 +41,8 @@ def search(
     return apps
 
 
-def get_job(job_id: str) -> _Job:
-    return _Job.create_from_uuid(uuid=job_id)
+def get_job(job_id: str, job_token: Optional[str] = None) -> _Job:
+    return _Job.create_from_uuid(uuid=job_id, auth_token=job_token)
 
 
 def get_data_record(uri: str) -> _DataRecord:

biolib/_data_record/data_record.py

@@ -6,19 +6,21 @@ from pathlib import Path
 from struct import Struct
 from typing import Callable, Dict, List, Optional, Union, cast
 
-from biolib import api, utils
+from biolib import api
 from biolib._internal import types
 from biolib._internal.data_record import get_data_record_state_from_uri
 from biolib._internal.data_record.data_record import validate_sqlite_v1
+from biolib._internal.data_record.push_data import (
+    push_data_path,
+    validate_data_path_and_get_files_and_size_of_directory,
+)
 from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
-from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib._internal.http_client import HttpClient
 from biolib.api import client as api_client
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersion, DataRecordVersionInfo
 from biolib.biolib_binary_format import LazyLoadedFile
 from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
-from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
 from biolib.utils.app_uri import parse_app_uri
 from biolib.utils.zip.remote_zip import RemoteZip
@@ -85,18 +87,8 @@ class DataRecord:
         self.download_files(output_dir=output_dir, path_filter=path_filter)
 
     def update(self, data_path: str, chunk_size_in_mb: Optional[int] = None) -> None:
-        assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
         BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Data Record')
-
-        if os.path.realpath(data_path) == '/':
-            raise BioLibError('Pushing your root directory is not possible')
-
-        original_working_dir = os.getcwd()
-        os.chdir(data_path)
-        files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
-
-        if data_size_in_bytes > 4_500_000_000_000:
-            raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+        files_to_zip, data_size_in_bytes = validate_data_path_and_get_files_and_size_of_directory(data_path)
 
         # validate data record
         detailed_dict: types.DataRecordDetailedDict = self._get_detailed_dict()
@@ -114,40 +106,23 @@ class DataRecord:
                 else:
                     raise Exception(f"Error processing data record validation: unknown rule type {rule['type']}")
 
-        min_chunk_size_bytes = 10_000_000
-        chunk_size_in_bytes: int
-        if chunk_size_in_mb:
-            chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
-            if chunk_size_in_bytes < min_chunk_size_bytes:
-                logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
-                chunk_size_in_bytes = min_chunk_size_bytes
-        else:
-            # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
-            chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
-
-        data_size_in_mb = round(data_size_in_bytes / 10**6)
-        logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
-
         response = api.client.post(path='/lfs/versions/', data={'resource_uuid': self._state['resource_uuid']})
         data_record_version: DataRecordVersion = response.json()
-        iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
-
-        multipart_uploader = utils.MultiPartUploader(
-            use_process_pool=True,
-            get_presigned_upload_url_request=dict(
-                headers=None,
-                requires_biolib_auth=True,
-                path=f"/lfs/versions/{data_record_version['uuid']}/presigned_upload_url/",
-            ),
-            complete_upload_request=dict(
-                headers=None,
-                requires_biolib_auth=True,
-                path=f"/lfs/versions/{data_record_version['uuid']}/complete_upload/",
-            ),
+        resource_version_uuid = data_record_version['uuid']
+
+        push_data_path(
+            data_path=data_path,
+            data_size_in_bytes=data_size_in_bytes,
+            files_to_zip=files_to_zip,
+            resource_version_uuid=resource_version_uuid,
+            chunk_size_in_mb=chunk_size_in_mb,
+        )
+
+        api.client.patch(
+            path=f'/resources/versions/{resource_version_uuid}/',
+            data={'state': 'published', 'set_as_active': True},
         )
 
-        multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
-        os.chdir(original_working_dir)
         logger.info(f"Successfully pushed a new Data Record version '{data_record_version['uri']}'")
         self._state = get_data_record_state_from_uri(data_record_version['uri'])
 
@@ -177,12 +152,11 @@ class DataRecord:
         data_record_info: DataRecordInfo = response.json()
         logger.info(f"Successfully created new Data Record '{data_record_info['uri']}'")
 
+        data_record = DataRecord.get_by_uri(uri=data_record_info['uri'])
         if data_path is not None:
-            data_record = DataRecord.get_by_uri(uri=data_record_info['uri'])
             data_record.update(data_path=data_path)
-            return data_record
-        else:
-            return DataRecord.get_by_uri(uri=data_record_info['uri'])
+
+        return data_record
 
     @staticmethod
     def fetch(uri: Optional[str] = None, count: Optional[int] = None) -> List['DataRecord']:

biolib/_internal/data_record/data_record.py

@@ -83,6 +83,8 @@ def verify_schema(specification: SqliteV1DatabaseSchema, actual_schema: SqliteV1
 
 def get_data_record_state_from_uri(uri) -> 'DataRecordVersionInfo':
     app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': uri}).json()
+    if app_response['app']['type'] != 'data-record':
+        raise Exception(f'Resource "{uri}" is not a Data Record')
     return DataRecordVersionInfo(
         resource_uri=app_response['app_version']['app_uri'],
         resource_uuid=app_response['app']['public_id'],

biolib/_internal/data_record/push_data.py

@@ -0,0 +1,67 @@
+import os
+
+from biolib import utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
+from biolib._internal.types.typing import List, Optional, Tuple
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger
+
+
+def validate_data_path_and_get_files_and_size_of_directory(data_path: str) -> Tuple[List[str], int]:
+    assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
+
+    if os.path.realpath(data_path) == '/':
+        raise BioLibError('Pushing your root directory is not possible')
+
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+    files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
+    os.chdir(original_working_dir)
+
+    if data_size_in_bytes > 4_500_000_000_000:
+        raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+
+    return files_to_zip, data_size_in_bytes
+
+
+def push_data_path(
+    data_path: str,
+    data_size_in_bytes: int,
+    files_to_zip: List[str],
+    resource_version_uuid: str,
+    chunk_size_in_mb: Optional[int] = None,
+) -> None:
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+
+    min_chunk_size_bytes = 10_000_000
+    chunk_size_in_bytes: int
+    if chunk_size_in_mb:
+        chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
+        if chunk_size_in_bytes < min_chunk_size_bytes:
+            logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
+            chunk_size_in_bytes = min_chunk_size_bytes
+    else:
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
+
+    data_size_in_mb = round(data_size_in_bytes / 10**6)
+    logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
+
+    iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
+    multipart_uploader = utils.MultiPartUploader(
+        use_process_pool=True,
+        get_presigned_upload_url_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f'/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
+        ),
+        complete_upload_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f'/lfs/versions/{resource_version_uuid}/complete_upload/',
+        ),
+    )
+
+    multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
+    os.chdir(original_working_dir)

biolib/_internal/http_client.py

@@ -125,12 +125,14 @@ class HttpClient:
 
             except urllib.error.URLError as error:
                 if isinstance(error.reason, socket.timeout):
-                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                    if retry_count > 0:
+                        logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                     last_error = error
                 else:
                     raise error
             except socket.timeout as error:
-                logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                if retry_count > 0:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                 last_error = error
 
         raise last_error or Exception(f'HTTP {method} request failed after {retries} retries for "{url}"')

biolib/_internal/push_application.py

@@ -6,6 +6,10 @@ import rich.progress
 import yaml
 
 from biolib import api, utils
+from biolib._internal.data_record.push_data import (
+    push_data_path,
+    validate_data_path_and_get_files_and_size_of_directory,
+)
 from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
@@ -94,10 +98,25 @@ def push_application(
     zip_filters.add('.biolib/config.yml')
 
     input_files_maps_to_root = False
+    app_data_path: Optional[Path] = None
     try:
         with open(config_yml_path) as config_yml_file:
             config = yaml.safe_load(config_yml_file.read())
 
+        app_data = config.get('app_data')
+        if app_data:
+            if not isinstance(app_data, str):
+                raise BioLibError(
+                    f'In .biolib/config.yml the value of "app_data" must be a string but got {type(app_data)}'
+                )
+
+            app_data_path = app_path_absolute.joinpath(app_data).resolve()
+            if not app_data_path.is_dir():
+                raise BioLibError(
+                    'In .biolib/config.yml the value of "app_data" must be a path to a directory '
+                    'in the application directory'
+                )
+
         license_file_relative_path = config.get('license_file', 'LICENSE')
         if app_path_absolute.joinpath(license_file_relative_path).is_file():
             zip_filters.add(license_file_relative_path)
@@ -118,11 +137,14 @@ def push_application(
             if module.get('source_files'):
                 zip_filters.add('*')
 
-            for mapping in module['input_files']:
+            for mapping in module.get('input_files', []):
                 mapping_parts = mapping.split(' ')
                 if len(mapping_parts) == 3 and mapping_parts[2] == '/':
                     input_files_maps_to_root = True
 
+    except BioLibError as error:
+        raise error from None
+
     except Exception as error:
         raise BioLibError('Failed to parse the .biolib/config.yml file') from error
 
@@ -175,6 +197,17 @@ def push_application(
         else None,
     )
 
+    if app_data_path:
+        app_data_files_to_zip, app_data_size_in_bytes = validate_data_path_and_get_files_and_size_of_directory(
+            data_path=str(app_data_path),
+        )
+        push_data_path(
+            resource_version_uuid=new_app_version_json['public_id'],
+            data_path=str(app_data_path),
+            data_size_in_bytes=app_data_size_in_bytes,
+            files_to_zip=app_data_files_to_zip,
+        )
+
     #  Don't push docker images if copying from another app version
     docker_tags = new_app_version_json.get('docker_tags', {})
     if not app_version_to_copy_images_from and docker_tags:
@@ -211,6 +244,11 @@ def push_application(
             try:
                 logger.info(f'Trying to push image {docker_image_name} defined on module {module_name}.')
                 image = docker_client.images.get(docker_image_name)
+                architecture = image.attrs.get('Architecture')
+                if architecture != 'amd64':
+                    print(f"Error: '{docker_image_name}' is compiled for {architecture}, expected x86 (amd64).")
+                    print('If you are on an ARM processor, try passing --platform linux/amd64 to docker build.')
+                    exit(1)
                 absolute_repo_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{repo}'
                 image.tag(absolute_repo_uri, tag)
 

biolib/_internal/runtime.py

@@ -7,6 +7,7 @@ class RuntimeJobDataDict(TypedDict):
     job_uuid: str
     job_auth_token: str
     app_uri: str
+    is_environment_biolib_cloud: bool
 
 
 class BioLibRuntimeError(Exception):

biolib/_internal/types/__init__.py

@@ -2,3 +2,4 @@ from .app import *  # noqa: F403
 from .data_record import *  # noqa: F403
 from .experiment import *  # noqa: F403
 from .resource import *  # noqa: F403
+from .resource_version import *  # noqa: F403

biolib/_internal/types/data_record.py

@@ -7,7 +7,7 @@ class SqliteV1ForeignKey(TypedDict):
 
 
 class SqliteV1Column(TypedDict):
-    type: Literal['INTEGER', 'REAL', 'TEXT', 'JSON']  # noqa:F821
+    type: Literal['INTEGER', 'REAL', 'TEXT', 'JSON']
     nullable: Optional[bool]
     foreign_key: Optional[SqliteV1ForeignKey]
     json_schema: Optional[Dict]

biolib/_internal/types/resource_version.py

@@ -0,0 +1,7 @@
+from .typing import Literal, TypedDict
+
+
+class ResourceVersionDetailedDict(TypedDict):
+    semantic_version: str
+    state: Literal['published', 'unpublished']
+    uuid: str

biolib/_internal/utils/multinode.py

@@ -0,0 +1,267 @@
+import glob
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+
+import biolib
+from biolib.utils import SeqUtil
+
+
+def natsorted(lst):
+    """Sort the list using the natural sort key."""
+
+    def _natural_sort_key(s):
+        """A key function for natural sorting."""
+        return [int(text) if text.isdigit() else text.lower() for text in re.split('([0-9]+)', s)]
+
+    return sorted(lst, key=_natural_sort_key)
+
+
+def fasta_above_threshold(fasta_file, work_threshold, work_per_residue=1, verbose=False):
+    """True if total FASYA residue work above max_work"""
+
+    records = SeqUtil.parse_fasta(fasta_file)
+
+    # Calculate work units
+    total_work_units = 0
+    for i, record in enumerate(records):
+        sequence_work_units = len(record.sequence) * work_per_residue
+        total_work_units += sequence_work_units
+
+        if total_work_units >= work_threshold:
+            if verbose:
+                print(f'FASTA above threshold (stopped at {total_work_units}) >= {work_threshold}')
+                print(f'From  from {i+1}/{len(records)} sequences in {fasta_file}')
+            return True
+
+    if verbose:
+        print(f'FASTA below threshold ({total_work_units}) < {work_threshold}')
+        print(f'From {len(records)} sequences in {fasta_file}')
+
+    return False
+
+
+def run_locally(command_list, args):
+    """Run script locally (no multi-node processing)"""
+
+    # Prepare command
+    new_args = vars(args)
+
+    # Delete multinode-specific input arguments
+    for k in list(new_args.keys()):
+        if str(k).startswith('multinode'):
+            del new_args[k]
+
+    # Convert to list format
+    new_args_list = _args_dict_to_args_list(new_args)
+
+    # Prepare command, e.g. ["python3", "predict.py"] + new_args_list
+    command = command_list + new_args_list
+
+    if args.verbose >= 1:
+        print(f'Running {command}')
+
+    # Run command
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode == 0:
+        print(f'{result.stdout}')
+    else:
+        print(f'Error: {result.stderr}')
+
+
+def fasta_batch_records(fasta_file, work_per_batch_min, work_per_residue=1, verbose=False):
+    """Converts FASTA records to batches of records, based on thresholds"""
+
+    def log_batches(batches):
+        for i, batch in enumerate(batches):
+            batch_dict = {
+                'records': len(batch),
+                'residues': sum(len(record.sequence) for record in batch),
+            }
+
+            n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+            print(f'Batch {i+1}: {n_res} residues from {n_seqs} sequences')
+
+    records = SeqUtil.parse_fasta(fasta_file)
+
+    batches = []
+    batch = []
+    current_work_units = 0
+    total_work_units = 0
+    for record in records:
+        # Add to batch
+        batch.append(record)
+
+        # Calculate work units
+        seq = record.sequence
+        sequence_work_units = len(seq) * work_per_residue
+
+        # Increase counters
+        current_work_units += sequence_work_units
+        total_work_units += sequence_work_units
+
+        # If above limit, start a new batch
+        if current_work_units >= work_per_batch_min:
+            batches.append(batch)
+            batch = []
+            current_work_units = 0
+
+    # Append last batch if present
+    if batch:
+        batches.append(batch)
+
+    if verbose:
+        log_batches(batches)
+
+    return batches
+
+
+def fasta_send_batches_biolib(
+    app_url, batches, args, args_fasta='fasta', machine='cpu.large', stream_all_jobs=True, verbose=1
+):
+    """
+    Send jobs through pybiolib interface
+    """
+
+    if args.verbose >= 1:
+        print(f'Sending {len(batches)} batches to Biolib')
+
+    # Login to biolib, prepare app
+    # current_app = biolib.load(Runtime.get_app_uri())
+    current_app = biolib.load(app_url)  # Nb: uses "_" not "-"
+
+    # Compute results
+    job_list = []
+    for i, batch_records in enumerate(batches):  # MH
+        # Write FASTA, send to server
+        with tempfile.TemporaryDirectory() as tempdir:
+            # New arguments
+            new_args = vars(args)
+
+            # Write batched FASTA to send
+            fasta_path = f'{tempdir}/input.fasta'
+            SeqUtil.write_records_to_fasta(fasta_path, batch_records)
+            new_args[args_fasta] = fasta_path
+            new_args['multinode_only_local'] = True
+
+            # Convert to list
+            new_args_list = _args_dict_to_args_list(new_args)
+
+            # Send job
+            job = current_app.cli(args=new_args_list, blocking=False, machine=machine)
+            job_list.append(job)
+
+            # Job stats
+            if args.verbose:
+                batch_dict = _get_batch_stats(batch_records)
+                n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+                print(f'Sending job {i+1}: {n_res} residues from {n_seqs} sequences -> arg_list = {new_args_list}')
+
+    # Stream job output at a time
+    print('Streaming job outputs ...')
+    for i, job in enumerate(job_list):
+        # Try to print if verbose. Always on first job, otherwise only if stream_all_jobs set
+        if (i == 0 and verbose) or (stream_all_jobs and verbose):
+            job.stream_logs()
+
+        # Check if job succeeded
+        assert job.get_exit_code() == 0, f'Job failed with exit code {job.get_exit_code()}'
+
+        # Write to disk
+        output_dir = f'job_output/job_{i+1}'
+        job.save_files(output_dir=output_dir)
+
+        if verbose:
+            print(f'Saving to {output_dir}')
+
+
+def merge_folder(folder_name, job_out_dir='job_output', out_dir='output', verbose=1):
+    """Helper function for merging folders"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_folder = f'{job_dirs[0]}/{folder_name}'
+    merged_folder = f'{out_dir}/{folder_name}'
+    shutil.move(first_folder, merged_folder)
+
+    if verbose:
+        print(f'Merging {folder_name} from {len(job_dirs)} directories to {merged_folder}')
+
+    # If more than one folder, merge to first
+    if len(job_dirs) >= 2:
+        # Find each job output file
+        for job_dir in job_dirs[1:]:
+            # Move over extra files
+            extra_folder = f'{job_dir}/{folder_name}'
+            extra_files = os.listdir(extra_folder)
+            for file_name in extra_files:
+                file_path = f'{extra_folder}/{file_name}'
+                shutil.move(file_path, merged_folder)
+
+
+def merge_file(
+    file_name,
+    header_lines_int=1,
+    job_out_dir='job_output',
+    out_dir='output',
+    verbose=1,
+):
+    """Helper function for merging files with headers"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_file = f'{job_dirs[0]}/{file_name}'
+    merged_file = f'{out_dir}/{file_name}'
+    shutil.move(first_file, merged_file)
+
+    if verbose:
+        print(f'Merging {file_name} from {len(job_dirs)} directories to {merged_file}')
+
+    # If more than one file, append to first
+    if len(job_dirs) >= 2:
+        # Open first file
+        with open(merged_file, 'a') as merged_file_handle:
+            # Find each job output file
+            for job_dir in job_dirs[1:]:
+                # Open extra file
+                extra_file = f'{job_dir}/{file_name}'
+                with open(extra_file) as extra_file_handle:
+                    # Skip first n header lines
+                    for _ in range(header_lines_int):
+                        next(extra_file_handle)
+
+                    # Append content to first file
+                    contents = extra_file_handle.read()
+                    merged_file_handle.write(contents)
+
+
+def _get_batch_stats(batch):
+    stats_dict = {
+        'records': len(batch),
+        'residues': sum(len(R.sequence) for R in batch),
+    }
+
+    return stats_dict
+
+
+def _args_dict_to_args_list(new_args):
+    """Converts args dict to list of arguments for Biolib"""
+
+    nested_list = [[f'--{key}', f'{value}'] for key, value in new_args.items()]
+
+    arg_list = []
+    for lst in nested_list:
+        for item in lst:
+            arg_list.append(item)
+
+    return arg_list

biolib/_runtime/runtime.py

@@ -14,6 +14,10 @@ class Runtime:
     def check_is_environment_biolib_app() -> bool:
         return bool(Runtime._try_to_get_job_data())
 
+    @staticmethod
+    def check_is_environment_biolib_cloud() -> bool:
+        return Runtime._get_job_data().get('is_environment_biolib_cloud', False)
+
     @staticmethod
     def get_job_id() -> str:
         return Runtime._get_job_data()['job_uuid']

biolib/biolib_api_client/app_types.py

@@ -32,6 +32,7 @@ class App(TypedDict):
     public_id: str
     state: str
     resource_uri: str
+    type: str
 
 
 class AppGetResponse(TypedDict):

biolib/cli/data_record.py

@@ -6,6 +6,7 @@ from typing import Dict, List
 import click
 
 from biolib._data_record.data_record import DataRecord
+from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import Optional
 
@@ -57,6 +58,7 @@ def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
 @click.argument('uri', required=True)
 @click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
 def describe(uri: str, output_as_json: bool) -> None:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='get Data Record description')
     record = DataRecord.get_by_uri(uri)
     files_info: List[Dict] = []
     total_size_in_bytes = 0

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -286,6 +286,7 @@ class DockerExecutor:
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
+                is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
             )
             secrets: Dict[str, str] = dict(
                 **module.get('secrets', {}),

biolib/compute_node/job_worker/job_storage.py

@@ -47,8 +47,21 @@ class JobStorage:
         module_output_path = os.path.join(job_temporary_dir, JobStorage.module_output_file_name)
         module_output_size = os.path.getsize(module_output_path)
 
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        max_chunk_count = 9_000
+        min_chunk_size_bytes = 50_000_000
+        chunk_size_in_bytes = max(min_chunk_size_bytes, module_output_size // max_chunk_count)
+
+        logger_no_user_data.debug(
+            f'Job "{job_uuid}" uploading result of size {module_output_size} bytes '
+            f'with chunk size of {chunk_size_in_bytes} bytes...'
+        )
+
         with open(module_output_path, mode='rb') as module_output_file:
-            module_output_iterator = get_chunk_iterator_from_file_object(module_output_file)
+            module_output_iterator = get_chunk_iterator_from_file_object(
+                file_object=module_output_file,
+                chunk_size_in_bytes=chunk_size_in_bytes,
+            )
             multipart_uploader = JobStorage._get_module_output_uploader(job_uuid)
             multipart_uploader.upload(
                 payload_iterator=module_output_iterator,

biolib/compute_node/remote_host_proxy.py

@@ -1,3 +1,4 @@
+import base64
 import io
 import subprocess
 import tarfile
@@ -160,6 +161,9 @@ class RemoteHostProxy:
             access_token = BiolibApiClient.get().access_token
             bearer_token = f'Bearer {access_token}' if access_token else ''
 
+            biolib_index_basic_auth = f'compute_node|admin:{compute_node_auth_token},{self._job_uuid}'
+            biolib_index_basic_auth_base64 = base64.b64encode(biolib_index_basic_auth.encode('utf-8')).decode('utf-8')
+
             nginx_config = f"""
 events {{
   worker_connections  1024;
@@ -270,7 +274,7 @@ http {{
         }}
 
         location /api/lfs/ {{
-            proxy_pass               https://$upstream_hostname/api/lfs/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
             proxy_set_header         job-uuid "{self._job_uuid}";
@@ -279,7 +283,16 @@ http {{
         }}
 
         location /api/app/ {{
-            proxy_pass               https://$upstream_hostname/api/app/;
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/resources/data-records/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
             proxy_set_header         job-uuid "{self._job_uuid}";
@@ -288,33 +301,40 @@ http {{
         }}
 
         location /api/ {{
-            proxy_pass               https://$upstream_hostname/api/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/job-storage/ {{
-            proxy_pass               https://$upstream_hostname/proxy/storage/job-storage/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/lfs/versions/ {{
-            proxy_pass               https://$upstream_hostname/proxy/storage/lfs/versions/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/cloud/ {{
-            proxy_pass               https://$upstream_hostname/proxy/cloud/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
+        location /proxy/index/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "Basic {biolib_index_basic_auth_base64}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
         location / {{
             return 404 "Not found";
         }}

biolib/compute_node/webserver/worker_thread.py

@@ -2,23 +2,23 @@ import base64
 import os
 import random
 import shutil
+import socket
 import sys
-import time
 import threading
-import socket
+import time
 from queue import Queue
 
-from biolib import utils
+from biolib import api, utils
+from biolib.biolib_binary_format import ModuleOutputV2, SystemException, SystemStatusUpdate
 from biolib.biolib_binary_format.utils import LocalFileIndexableBuffer
+from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker import JobWorkerProcess
 from biolib.compute_node.job_worker.job_storage import JobStorage
 from biolib.compute_node.socker_listener_thread import SocketListenerThread
 from biolib.compute_node.socket_sender_thread import SocketSenderThread
+from biolib.compute_node.utils import SystemExceptionCodes, WorkerThreadException, get_package_type
 from biolib.compute_node.webserver import webserver_utils
-from biolib.biolib_binary_format import SystemStatusUpdate, SystemException, ModuleOutputV2
-from biolib.compute_node.utils import get_package_type, WorkerThreadException, SystemExceptionCodes
-from biolib.biolib_logging import logger, logger_no_user_data
 
 SOCKET_HOST = '127.0.0.1'
 
@@ -37,7 +37,7 @@ class WorkerThread(threading.Thread):
             self._sender_thread = None
             self._start_and_connect_to_compute_process()
 
-            logger.debug(f"WorkerThread connected to port {self._socket_port}")
+            logger.debug(f'WorkerThread connected to port {self._socket_port}')
 
         except Exception as exception:
             logger_no_user_data.error(exception)
@@ -79,20 +79,16 @@ class WorkerThread(threading.Thread):
                     if progress == 94:
                         # Get Job exit code
                         try:
-                            module_output_path = os.path.join(self._job_temporary_dir,
-                                                              JobStorage.module_output_file_name)
-                            module_output = ModuleOutputV2(
-                                buffer=LocalFileIndexableBuffer(
-                                    filename=module_output_path
-                                )
+                            module_output_path = os.path.join(
+                                self._job_temporary_dir,
+                                JobStorage.module_output_file_name,
                             )
+                            module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
                             self.compute_state['exit_code'] = module_output.get_exit_code()
                             logger_no_user_data.debug(f"Got exit code: {self.compute_state['exit_code']}")
 
                         except Exception as error:  # pylint: disable=broad-except
-                            logger_no_user_data.error(
-                                f'Could not get exit_code from module output due to: {error}'
-                            )
+                            logger_no_user_data.error(f'Could not get exit_code from module output due to: {error}')
 
                         if utils.IS_RUNNING_IN_CLOUD:
                             JobStorage.upload_module_output(
@@ -107,7 +103,7 @@ class WorkerThread(threading.Thread):
                 elif package_type == 'SystemException':
                     error_code = SystemException(package).deserialize()
                     self.compute_state['status']['error_code'] = error_code
-                    logger.debug("Hit error. Terminating Worker Thread and Compute Process")
+                    logger.debug('Hit error. Terminating Worker Thread and Compute Process')
                     self.compute_state['progress'] = 95
                     self.terminate()
 
@@ -153,10 +149,10 @@ class WorkerThread(threading.Thread):
 
         # Starting a thread for accepting connections before starting the process that should to connect to the socket
         logger_no_user_data.debug('Starting connection thread')
-        self._connection_thread = threading.Thread(target=self._accept_new_socket_connection, args=[
-            received_messages_queue,
-            messages_to_send_queue
-        ])
+        self._connection_thread = threading.Thread(
+            target=self._accept_new_socket_connection,
+            args=[received_messages_queue, messages_to_send_queue],
+        )
         self._connection_thread.start()
         logger_no_user_data.debug('Started connection thread')
         logger_no_user_data.debug('Starting compute process')
@@ -177,6 +173,16 @@ class WorkerThread(threading.Thread):
         self._sender_thread.start()
 
     def terminate(self) -> None:
+        cloud_job_uuid = self.compute_state['cloud_job_id']
+        exit_code = self.compute_state.get('exit_code')
+        system_exception_code = self.compute_state['status'].get('error_code')
+        if utils.IS_RUNNING_IN_CLOUD:
+            CloudUtils.finish_cloud_job(
+                cloud_job_id=cloud_job_uuid,
+                system_exception_code=system_exception_code,
+                exit_code=exit_code,
+            )
+
         deregistered_due_to_error = False
         if self._job_worker_process:
             logger_no_user_data.debug(
@@ -184,7 +190,8 @@ class WorkerThread(threading.Thread):
             )
             self._job_worker_process.terminate()
 
-            for _ in range(10):
+            clean_up_timeout_in_seconds = 600
+            for _ in range(clean_up_timeout_in_seconds):
                 if self._job_worker_process.exitcode is not None:
                     logger_no_user_data.debug(
                         f'Job "{self._job_uuid}" worker process exitcode {self._job_worker_process.exitcode}'
@@ -196,28 +203,18 @@ class WorkerThread(threading.Thread):
 
             if self._job_worker_process.exitcode is None:
                 # TODO: Figure out if more error handling is necessary here
-                logger_no_user_data.error(f'Job {self._job_uuid} worker process did not exit within 10 seconds')
+                logger_no_user_data.error(
+                    f'Job {self._job_uuid} worker process did not exit within {clean_up_timeout_in_seconds} seconds'
+                )
                 if utils.IS_RUNNING_IN_CLOUD:
                     logger_no_user_data.error('Deregistering compute node...')
                     CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
         # Delete result as error occurred
-        system_exception_code = self.compute_state['status'].get('error_code')
         if system_exception_code and os.path.exists(self._job_temporary_dir):
             shutil.rmtree(self._job_temporary_dir)
 
-        exit_code = self.compute_state.get('exit_code')
-
-        if utils.IS_RUNNING_IN_CLOUD:
-            # Get and send compute node exception code and job exit code if present
-            logger_no_user_data.debug(f"Sending exit code {exit_code}")
-            CloudUtils.finish_cloud_job(
-                cloud_job_id=self.compute_state['cloud_job_id'],
-                system_exception_code=system_exception_code,
-                exit_code=exit_code
-            )
-
         if self._socket:
             self._socket.close()
 
@@ -225,7 +222,7 @@ class WorkerThread(threading.Thread):
             self._connection.close()
 
         if self.compute_state['progress'] == 95:
-            seconds_to_sleep = 60  # 1 minute
+            seconds_to_sleep = 5
             logger_no_user_data.debug(
                 f'Job "{self._job_uuid}" worker thread sleeping for {seconds_to_sleep} seconds before cleaning up'
             )
@@ -234,7 +231,7 @@ class WorkerThread(threading.Thread):
 
         compute_state_dict = webserver_utils.JOB_ID_TO_COMPUTE_STATE_DICT
         if self._job_uuid in compute_state_dict:
-            # Delete result as user has not started download within 60 seconds
+            # Delete result as user has not started download
             if compute_state_dict[self._job_uuid]['progress'] == 95 and os.path.exists(self._job_temporary_dir):
                 shutil.rmtree(self._job_temporary_dir)
 
@@ -245,12 +242,18 @@ class WorkerThread(threading.Thread):
                 f'Job "{self._job_uuid}" could not be found, maybe it has already been cleaned up'
             )
 
-        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread terminated')
-
         if utils.IS_RUNNING_IN_CLOUD:
+            config = CloudUtils.get_webserver_config()
+            logger_no_user_data.debug(f'Job "{self._job_uuid}" reporting CloudJob "{cloud_job_uuid}" as cleaned up...')
+            api.client.post(
+                path=f'/internal/compute-nodes/cloud-jobs/{cloud_job_uuid}/cleaned-up/',
+                headers={'Compute-Node-Auth-Token': config['compute_node_info']['auth_token']},
+            )
+
             if deregistered_due_to_error:
                 CloudUtils.shutdown()  # shutdown now
             else:
                 webserver_utils.update_auto_shutdown_time()
 
+        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread exiting...')
         sys.exit()

biolib/utils/seq_util.py

@@ -1,7 +1,7 @@
 import re
-from io import BufferedIOBase
+from io import BufferedIOBase, TextIOBase
 
-from biolib.typing_utils import Dict, List, Optional, Union
+from biolib.typing_utils import Dict, List, Optional, Union, Iterator
 
 
 class SeqUtilRecord:
@@ -37,67 +37,77 @@ class SeqUtil:
         allow_any_sequence_characters: bool = False,
         allow_empty_sequence: bool = True,
         file_name: Optional[str] = None,
-    ) -> List[SeqUtilRecord]:
+) -> Iterator[SeqUtilRecord]:
+        def process_and_yield_record(header: str, sequence_lines: List[str]):
+            sequence = ''.join(sequence_lines)
+            sequence_id = header.split()[0]
+            if not allow_any_sequence_characters:
+                invalid_sequence_characters = SeqUtil._find_invalid_sequence_characters(sequence)
+                if invalid_sequence_characters:
+                    raise Exception(
+                        f'Error: Invalid character ("{invalid_sequence_characters[0]}") found in sequence {sequence_id}'
+                    )
+            if not allow_empty_sequence and not sequence:
+                raise Exception(f'Error: No sequence found for fasta entry {sequence_id}')
+            yield SeqUtilRecord(
+                sequence=sequence,
+                sequence_id=sequence_id,
+                description=header[len(sequence_id):].strip()
+            )
+
+        def line_generator_from_buffered_io_base(file_handle: BufferedIOBase) -> Iterator[str]:
+            for line in file_handle:
+                yield line.decode('utf-8')
+
+        def line_generator_from_text_io_base(file_handle: TextIOBase) -> Iterator[str]:
+            for line in file_handle:
+                yield line
+
         if input_file is None:
             if file_name:
                 input_file = file_name
             else:
                 raise ValueError('input_file must be a file name (str) or a BufferedIOBase object')
+
+        file_handle = None
         if isinstance(input_file, str):
-            with open(input_file) as file_handle:
-                data = file_handle.read().strip()
+            file_handle = open(input_file, "rb")
+            line_iterator = line_generator_from_buffered_io_base(file_handle)
         elif isinstance(input_file, BufferedIOBase):
-            data = input_file.read().decode('utf-8')
+            line_iterator = line_generator_from_buffered_io_base(input_file)
+        elif isinstance(input_file, TextIOBase):
+            line_iterator = line_generator_from_text_io_base(input_file)
         else:
             raise ValueError('input_file must be a file name (str) or a BufferedIOBase object')
-        if not data:
-            return []
-
-        if '>' not in data:
-            if default_header:
-                lines_with_header = []
-                for index, line in enumerate(data.split('\n')):
-                    index_string = str(index + 1) if index > 0 else ''
-                    lines_with_header.append(f'>{default_header}{index_string}\n{line}')
-
-                data = '\n'.join(lines_with_header)
-            else:
-                raise Exception(f'No header line found in FASTA file "{file_name}"')
-
-        splitted = []
-        tmp_data = ''
-        for line in data.splitlines():
-            if line.startswith('>'):
-                if tmp_data:
-                    splitted.append(tmp_data)
-                tmp_data = line[1:].strip() + '\n'
-            else:
-                if line.strip():
-                    tmp_data += line.strip() + '\n'
-
-        if tmp_data:
-            splitted.append(tmp_data)
-
-        parsed_sequences = []
-        for sequence_data in splitted:
-            sequence_data_splitted = sequence_data.strip().split('\n')
-            header_line = sequence_data_splitted[0].split()
-            sequence_id = header_line[0]
-            description = sequence_data_splitted[0][len(sequence_id) :].strip()
-            sequence = ''.join([seq.strip() for seq in sequence_data_splitted[1:]])
-
-            if not allow_any_sequence_characters:
-                invalid_sequence_characters = SeqUtil._find_invalid_sequence_characters(sequence)
-                if len(invalid_sequence_characters) > 0:
-                    raise Exception(
-                        f'Error: Invalid character ("{invalid_sequence_characters[0]}") found in sequence {sequence_id}'
-                    )
-            if not allow_empty_sequence and len(sequence) == 0:
-                raise Exception(f'Error: No sequence found for fasta entry {sequence_id}')
-
-            parsed_sequences.append(SeqUtilRecord(sequence=sequence, sequence_id=sequence_id, description=description))
 
-        return parsed_sequences
+        header = None
+        sequence_lines: List[str] = []
+
+        try:
+            for line_number, line in enumerate(line_iterator):
+                line = line.strip()
+                if not line:
+                    continue # skip empty lines
+                if line.startswith('>'):
+                    if header is not None:
+                        yield from process_and_yield_record(header, sequence_lines)
+
+                    header = line[1:].strip()
+                    sequence_lines = []
+                else:
+                    if header is None:
+                        if default_header:
+                            yield from process_and_yield_record(f"{default_header}{line_number}", [line])
+                        else:
+                            raise Exception(f'No header line found in FASTA file "{file_name}"')
+                    else:
+                        sequence_lines.append(line)
+
+            if header is not None:
+                yield from process_and_yield_record(header, sequence_lines)
+        finally:
+            if file_handle:
+                file_handle.close()
 
     @staticmethod
     def write_records_to_fasta(file_name: str, records: List[SeqUtilRecord]) -> None:

{pybiolib-1.1.2193.dist-info → pybiolib-1.2.105.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.2193
+Version: 1.2.105.dev1
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.2193.dist-info → pybiolib-1.2.105.dev1.dist-info}/RECORD

@@ -1,29 +1,32 @@
 LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
-biolib/__init__.py,sha256=_tThyzISH81yS9KXP_X3qEiKXmsIp5XOBcJIODfLVnc,4338
-biolib/_data_record/data_record.py,sha256=CoyYRse5VdUBhQzzPfR9BkytgOsM-IZxkfMX1kyRnPk,12589
+PYPI_README.md,sha256=_IH7pxFiqy2bIAmaVeA-iVTyUwWRjMIlfgtUbYTtmls,368
+biolib/__init__.py,sha256=q_YhAYw51Vq16IKtSk8_MJclDAa4CfCmPhvWDYmrSIg,4393
+biolib/_data_record/data_record.py,sha256=zVAhFU1RLI1-ptoQ_l639RNwrMANXV9j75yXHvB7dtA,10950
 biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/_internal/data_record/__init__.py,sha256=fGdME6JGRU_2VxpJbYpGXYndjN-feUkmKY4fuMyq3cg,76
-biolib/_internal/data_record/data_record.py,sha256=YmaAABR57goDCE8-rKb2j0FPMSbDtRPCm_HhT3mM074,4299
+biolib/_internal/data_record/data_record.py,sha256=g_-jdy5-Zem3dthwxJj2OuQqkDGTyc-iGqN1rtYYD1A,4418
+biolib/_internal/data_record/push_data.py,sha256=-L3a_7zZzDCXabBu3O4lWPMAMeBbeRPTrBlEM-_5SCI,2693
 biolib/_internal/data_record/remote_storage_endpoint.py,sha256=eCptuZ4DMAPnaNCVDvpWXwXGI6Jac9U1N5dqU8Cj95Q,1732
 biolib/_internal/file_utils.py,sha256=4jT6j7bB21c0JNn5BfnyWQib_zt0CVtJ_TiOFOStRcE,2604
 biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
 biolib/_internal/fuse_mount/experiment_fuse_mount.py,sha256=08aUdEq_bvqLBft_gSLjOClKDy5sBnMts1RfJf7AP_U,7012
-biolib/_internal/http_client.py,sha256=DdooXei93JKGYGV4aQmzue_oFzvHkozg2UCxgk9dfDM,5081
+biolib/_internal/http_client.py,sha256=Q7z7_DMI1EaVJSyevrra63piuee-NfqcAHRtQt76ZdA,5165
 biolib/_internal/lfs/__init__.py,sha256=gSWo_xg61UniYgD7yNYxeT4I9uaXBCBSi3_nmZjnPpE,35
 biolib/_internal/lfs/cache.py,sha256=pQS2np21rdJ6I3DpoOutnzPHpLOZgUIS8TMltUJk_k4,2226
 biolib/_internal/libs/__init__.py,sha256=Jdf4tNPqe_oIIf6zYml6TiqhL_02Vyqwge6IELrAFhw,98
 biolib/_internal/libs/fusepy/__init__.py,sha256=AWDzNFS-XV_5yKb0Qx7kggIhPzq1nj_BZS5y2Nso08k,41944
-biolib/_internal/push_application.py,sha256=8P7eXvySn7CRp5XBDkO3xjTGixS8g7-jD-_iwzM_XDI,10020
-biolib/_internal/runtime.py,sha256=9pZ3s3L7LGxdqOgnHh1KK3Jjyn_9MjhQmKHI-6hMT3U,448
-biolib/_internal/types/__init__.py,sha256=11ZucS8jKeLGAAswXyKI7FH2KLHd6T9Sh8ZK2Ar3jlk,152
+biolib/_internal/push_application.py,sha256=mKs3kIKW-ZYfz3Cy6LIyFBwsWkbcGZ9zgMk-xn5NDyg,11660
+biolib/_internal/runtime.py,sha256=BiHl4klUHr36MCpqKaUso4idHeBZfPAahLYRQrabFqA,486
+biolib/_internal/types/__init__.py,sha256=xLgOQJFh3GRtiqIJq7MaqHReZx4pp34_zcaFQ_JjuJ4,198
 biolib/_internal/types/app.py,sha256=Mz2QGD_jESX-K9JYnLWPo4YA__Q_1FQQTk9pvidCohU,118
-biolib/_internal/types/data_record.py,sha256=AHoIiwVqeHj0HozQxFRAyxk-d3XJgLWno4ic1z9eTrQ,865
+biolib/_internal/types/data_record.py,sha256=9r_vdhVs60YTnzU4XQFXfDrfS2P2MqD3BH2xa7lk6ck,852
 biolib/_internal/types/experiment.py,sha256=D94iBdn2nS92lRW-TOs1a2WKXJD5ZtmzL4ypggKX2ys,176
 biolib/_internal/types/resource.py,sha256=G-vPkZoe4Um6FPxsQZtRzAlbSW5sDW4NFkbjn21I3V4,372
+biolib/_internal/types/resource_version.py,sha256=sLxViYXloDDUhTDFgjegiQCj097OM1Ih1-uqlC_4ULA,174
 biolib/_internal/types/typing.py,sha256=D4EKKEe7kDx0K6lJi-H_XLtk-8w6nu2fdqn9bvzI-Xo,288
 biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
-biolib/_runtime/runtime.py,sha256=daYxzIpRoW4k-HJFu2BMXeylYSlCXn3-SqdSriCFnKw,2770
+biolib/_internal/utils/multinode.py,sha256=zWrQhcVK5u_xdWX2oIM-D_2fINqNPlqF_h71fu4K8LY,8279
+biolib/_runtime/runtime.py,sha256=oVgTnDDJv9L4BUP1_sd0oAj4LLyyiPSQdhp7ixWARvw,2923
 biolib/api/__init__.py,sha256=mQ4u8FijqyLzjYMezMUUbbBGNB3iFmkNdjXnWPZ7Jlw,138
 biolib/api/client.py,sha256=FRpdH5aI187b_I_4HUNi680v4iOP65z5f2RcUo8D8MA,3559
 biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
@@ -31,7 +34,7 @@ biolib/app/app.py,sha256=P2RwaDAskUHzlciuTJUroqUocRwoyOLT6YbgMyCRRDI,8484
 biolib/app/search_apps.py,sha256=K4a41f5XIWth2BWI7OffASgIsD0ko8elCax8YL2igaY,1470
 biolib/biolib_api_client/__init__.py,sha256=E5EMa19wJoblwSdQPYrxc_BtIeRsAuO0L_jQweWw-Yk,182
 biolib/biolib_api_client/api_client.py,sha256=ciNx4ybpyKG5LEf4KQdGEz13r0jTxImyQat4_HDecD0,7373
-biolib/biolib_api_client/app_types.py,sha256=FxSr4UqfnMhLe34p8bm02wsC3g1Jz8iaing5tRKDOQI,2442
+biolib/biolib_api_client/app_types.py,sha256=1sXz9XnLRKNALMglNdTbew7AL6OkcUan0MPdj4xQLis,2456
 biolib/biolib_api_client/auth.py,sha256=kjm0ZHnH3I8so3su2sZbBxNHYp-ZUdrZ5lwQ0K36RSw,949
 biolib/biolib_api_client/biolib_app_api.py,sha256=DndlVxrNTes6DOaWyMINLGZQCRMWVvR7gwt5HVlyf5Y,4240
 biolib/biolib_api_client/biolib_job_api.py,sha256=7bKfav3-12ewXkEUoLdCmbWdebW8148kxfGJW9SsXZI,7125
@@ -57,7 +60,7 @@ biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
 biolib/biolib_logging.py,sha256=J3E5H_LL5k6ZUim2C8gqN7E6lCBZMTpO4tnMpOPwG9U,2854
 biolib/cli/__init__.py,sha256=0v3c_J-U0k46c5ZWeQjLG_kTaKDJm81LBxQpDO2B_aI,1286
 biolib/cli/auth.py,sha256=rpWGmXs6Fz6CGrO9K8ibPRszOdXG78Vig_boKaVCD9A,2082
-biolib/cli/data_record.py,sha256=08JbZkFWKMo0PrnhhG0jQEKnNW7pPLti9cOw8s1TWfI,3344
+biolib/cli/data_record.py,sha256=t8DfJK2EZ_SNZ9drDA_N5Jqy8DNwf9f5SlFrIaOvtv0,3501
 biolib/cli/download_container.py,sha256=HIZVHOPmslGE5M2Dsp9r2cCkAEJx__vcsDz5Wt5LRos,483
 biolib/cli/init.py,sha256=wQOfii_au-d30Hp7DdH-WVw-WVraKvA_zY4za1w7DE8,821
 biolib/cli/lfs.py,sha256=z2qHUwink85mv9yDgifbVKkVwuyknGhMDTfly_gLKJM,4151
@@ -74,19 +77,19 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=2H7GooL0oAifPcbie0unatB4fRoHyqbsr6S91uagc_g,27952
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=LcYc4x4-vkmRM1FN1SNj7xRj4qZUNVjQWtgbYMFg3JU,28029
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
 biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
 biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
-biolib/compute_node/job_worker/job_storage.py,sha256=LNkklckDLbYgCHsK5FGrEK75Kw-H4f4JcTCAtuE9His,4035
+biolib/compute_node/job_worker/job_storage.py,sha256=lScHI3ubcHKagSEW243tgbIWXUfbWDHDjEOPMvXxJE8,4603
 biolib/compute_node/job_worker/job_worker.py,sha256=fuWoYJo9HOqLmWl8yeCXh0mhT4ebbkrWac-BVb58khs,28842
 biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
 biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO33vWtG60xg,1282
-biolib/compute_node/remote_host_proxy.py,sha256=CNWJLXXYm8DGujxEJIsg1wUKFoSgU0nhwdmjMn5gelE,14690
+biolib/compute_node/remote_host_proxy.py,sha256=eTT7x7ht_cxXMQ-0yXCvhKZW6mKeYM4KrfBf75KTbc8,15651
 biolib/compute_node/socker_listener_thread.py,sha256=T5_UikA3MB9bD5W_dckYLPTgixh72vKUlgbBvj9dbM0,1601
 biolib/compute_node/socket_sender_thread.py,sha256=YgamPHeUm2GjMFGx8qk-99WlZhEs-kAb3q_2O6qByig,971
 biolib/compute_node/utils.py,sha256=M7i_WTyxbFM3Lri9RWZ_8FeQNYrQIWpKGLfp2I55oeY,4677
@@ -95,7 +98,7 @@ biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLU
 biolib/compute_node/webserver/webserver.py,sha256=15PkRyhtdtSgFDxa0z78aPO4ciZURsFqJYi-HtUmZF8,6494
 biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
 biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
-biolib/compute_node/webserver/worker_thread.py,sha256=26tG73TADnOcXsAr7Iyf6smrLlCqB4x-vvmpUb8WqnA,11569
+biolib/compute_node/webserver/worker_thread.py,sha256=GRRBUqXdMKvbjyLQhYlqGIbFKeU2iiEXIe5IXi9wgdg,11806
 biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/experiments/experiment.py,sha256=jIRixmQm3Gq9YdJ3I0-rE1vFukXqq6U4zXehFOJ1yZk,7614
 biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
@@ -114,10 +117,10 @@ biolib/utils/__init__.py,sha256=fwjciJyJicvYyZcVTzfDBgD0SKY13DeXqvTeG4qZIy8,5548
 biolib/utils/app_uri.py,sha256=Yq_-_VGugQhMMo6mM5f0G9yNlLkr0WK4j0Nrf3FE4xQ,2171
 biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3100
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
-biolib/utils/seq_util.py,sha256=ZQFcaE37B2dtucN2zDjOmdya_X0ITc1zBFZJNQY13XA,5183
+biolib/utils/seq_util.py,sha256=WJnU9vZdwY8RHXvzATyV80OXzyJ7w9EkG33Tna9Nr6A,5698
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.2193.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.2193.dist-info/METADATA,sha256=NCFsHsCnbnOhw1O2L-polZfFa2pNdtA89TfBW7f_o7E,1508
-pybiolib-1.1.2193.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.2193.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.2193.dist-info/RECORD,,
+pybiolib-1.2.105.dev1.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.105.dev1.dist-info/METADATA,sha256=8gXSVU8uvqONlUb6KfFDPrrnjV4z9abxHxZ1FL4EtTo,1512
+pybiolib-1.2.105.dev1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.2.105.dev1.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.105.dev1.dist-info/RECORD,,