pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/_internal/utils/job_url.py

@@ -0,0 +1,33 @@
+import re
+from urllib.parse import urlparse
+
+import biolib.utils
+from biolib.typing_utils import Optional, Tuple
+
+
+def parse_result_id_or_url(result_id_or_url: str, default_token: Optional[str] = None) -> Tuple[str, Optional[str]]:
+    result_id_or_url = result_id_or_url.strip()
+
+    if '/' not in result_id_or_url:
+        return (result_id_or_url, default_token)
+
+    if not result_id_or_url.startswith('http://') and not result_id_or_url.startswith('https://'):
+        result_id_or_url = 'https://' + result_id_or_url
+
+    parsed_url = urlparse(result_id_or_url)
+
+    if biolib.utils.BIOLIB_BASE_URL:
+        expected_base = urlparse(biolib.utils.BIOLIB_BASE_URL)
+        if parsed_url.scheme != expected_base.scheme or parsed_url.netloc != expected_base.netloc:
+            raise ValueError(f'URL must start with {biolib.utils.BIOLIB_BASE_URL}, got: {result_id_or_url}')
+
+    pattern = r'/results?/(?P<uuid>[a-f0-9-]+)/?(?:\?token=(?P<token>[^&]+))?'
+    match = re.search(pattern, result_id_or_url, re.IGNORECASE)
+
+    if not match:
+        raise ValueError(f'URL must be in format <base_url>/results/<UUID>/?token=<token>, got: {result_id_or_url}')
+
+    uuid = match.group('uuid')
+    token = match.group('token') or default_token
+
+    return (uuid, token)

biolib/_internal/utils/multinode.py

@@ -0,0 +1,263 @@
+import glob
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+
+import biolib
+from biolib.utils import SeqUtil
+
+
+def natsorted(lst):
+    """Sort the list using the natural sort key."""
+
+    def _natural_sort_key(s):
+        """A key function for natural sorting."""
+        return [int(text) if text.isdigit() else text.lower() for text in re.split('([0-9]+)', s)]
+
+    return sorted(lst, key=_natural_sort_key)
+
+
+def fasta_above_threshold(fasta_file, work_threshold, work_per_residue=1, verbose=False):
+    """True if total FASYA residue work above max_work"""
+
+    records = SeqUtil.parse_fasta(fasta_file)
+
+    # Calculate work units
+    total_work_units = 0
+    for i, record in enumerate(records):
+        sequence_work_units = len(record.sequence) * work_per_residue
+        total_work_units += sequence_work_units
+
+        if total_work_units >= work_threshold:
+            if verbose:
+                print(f'FASTA above threshold (stopped at {total_work_units}) >= {work_threshold}')
+                print(f'From  from {i+1}/{len(records)} sequences in {fasta_file}')
+            return True
+
+    if verbose:
+        print(f'FASTA below threshold ({total_work_units}) < {work_threshold}')
+        print(f'From {len(records)} sequences in {fasta_file}')
+
+    return False
+
+
+def run_locally(command_list, args):
+    """Run script locally (no multi-node processing)"""
+
+    # Prepare command
+    new_args = vars(args)
+
+    # Delete multinode-specific input arguments
+    for k in list(new_args.keys()):
+        if str(k).startswith('multinode'):
+            del new_args[k]
+
+    # Convert to list format
+    new_args_list = _args_dict_to_args_list(new_args)
+
+    # Prepare command, e.g. ["python3", "predict.py"] + new_args_list
+    command = command_list + new_args_list
+
+    if args.verbose >= 1:
+        print(f'Running {command}')
+
+    # Run command
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode == 0:
+        print(f'{result.stdout}')
+    else:
+        print(f'Error: {result.stderr}')
+
+
+def fasta_batch_records(records, work_per_batch_min, work_per_residue=1, verbose=False):
+    """Converts FASTA records to batches of records, based on thresholds"""
+
+    def log_batches(batches):
+        for i, batch in enumerate(batches):
+            batch_dict = {
+                'records': len(batch),
+                'residues': sum(len(record.sequence) for record in batch),
+            }
+
+            n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+            print(f'Batch {i+1}: {n_res} residues from {n_seqs} sequences')
+
+    batches = []
+    batch = []
+    current_longest_seq_len = 0
+    for record in records:
+        seq_len = len(record.sequence)
+        potential_longest_seq_len = max(current_longest_seq_len, seq_len)
+
+        # Calculate work units if we were to add this record
+        potential_work_units = potential_longest_seq_len * work_per_residue * (len(batch) + 1)
+
+        if potential_work_units >= work_per_batch_min and len(batch) > 0:
+            batches.append(batch)
+            batch = []
+            current_longest_seq_len = 0
+            potential_longest_seq_len = seq_len
+
+        # Add to batch
+        batch.append(record)
+        current_longest_seq_len = potential_longest_seq_len
+
+    # Append last batch if present
+    if batch:
+        batches.append(batch)
+
+    if verbose:
+        log_batches(batches)
+
+    return batches
+
+
+def fasta_send_batches_biolib(
+    app_url, batches, args, args_fasta='fasta', machine='cpu.large', stream_all_jobs=True, verbose=1
+):
+    """
+    Send jobs through pybiolib interface
+    """
+
+    if args.verbose >= 1:
+        print(f'Sending {len(batches)} batches to Biolib')
+
+    # Login to biolib, prepare app
+    # current_app = biolib.load(Runtime.get_app_uri())
+    current_app = biolib.load(app_url)  # Nb: uses "_" not "-"
+
+    # Compute results
+    job_list = []
+    for i, batch_records in enumerate(batches):  # MH
+        # Write FASTA, send to server
+        with tempfile.TemporaryDirectory() as tempdir:
+            # New arguments
+            new_args = vars(args)
+
+            # Write batched FASTA to send
+            fasta_path = f'{tempdir}/input.fasta'
+            SeqUtil.write_records_to_fasta(fasta_path, batch_records)
+            new_args[args_fasta] = fasta_path
+            new_args['multinode_only_local'] = True
+
+            # Convert to list
+            new_args_list = _args_dict_to_args_list(new_args)
+
+            # Send job
+            job = current_app.cli(args=new_args_list, blocking=False, machine=machine)
+            job_list.append(job)
+
+            # Job stats
+            if args.verbose:
+                batch_dict = _get_batch_stats(batch_records)
+                n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+                print(f'Sending job {i+1}: {n_res} residues from {n_seqs} sequences -> arg_list = {new_args_list}')
+
+    # Stream job output at a time
+    print('Streaming job outputs ...')
+    for i, job in enumerate(job_list):
+        # Try to print if verbose. Always on first job, otherwise only if stream_all_jobs set
+        if (i == 0 and verbose) or (stream_all_jobs and verbose):
+            job.stream_logs()
+
+        # Check if job succeeded
+        assert job.get_exit_code() == 0, f'Job failed with exit code {job.get_exit_code()}'
+
+        # Write to disk
+        output_dir = f'job_output/job_{i+1}'
+        job.save_files(output_dir=output_dir)
+
+        if verbose:
+            print(f'Saving to {output_dir}')
+
+
+def merge_folder(folder_name, job_out_dir='job_output', out_dir='output', verbose=1):
+    """Helper function for merging folders"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_folder = f'{job_dirs[0]}/{folder_name}'
+    merged_folder = f'{out_dir}/{folder_name}'
+    shutil.move(first_folder, merged_folder)
+
+    if verbose:
+        print(f'Merging {folder_name} from {len(job_dirs)} directories to {merged_folder}')
+
+    # If more than one folder, merge to first
+    if len(job_dirs) >= 2:
+        # Find each job output file
+        for job_dir in job_dirs[1:]:
+            # Move over extra files
+            extra_folder = f'{job_dir}/{folder_name}'
+            extra_files = os.listdir(extra_folder)
+            for file_name in extra_files:
+                file_path = f'{extra_folder}/{file_name}'
+                shutil.move(file_path, merged_folder)
+
+
+def merge_file(
+    file_name,
+    header_lines_int=1,
+    job_out_dir='job_output',
+    out_dir='output',
+    verbose=1,
+):
+    """Helper function for merging files with headers"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_file = f'{job_dirs[0]}/{file_name}'
+    merged_file = f'{out_dir}/{file_name}'
+    shutil.move(first_file, merged_file)
+
+    if verbose:
+        print(f'Merging {file_name} from {len(job_dirs)} directories to {merged_file}')
+
+    # If more than one file, append to first
+    if len(job_dirs) >= 2:
+        # Open first file
+        with open(merged_file, 'a') as merged_file_handle:
+            # Find each job output file
+            for job_dir in job_dirs[1:]:
+                # Open extra file
+                extra_file = f'{job_dir}/{file_name}'
+                with open(extra_file) as extra_file_handle:
+                    # Skip first n header lines
+                    for _ in range(header_lines_int):
+                        next(extra_file_handle)
+
+                    # Append content to first file
+                    contents = extra_file_handle.read()
+                    merged_file_handle.write(contents)
+
+
+def _get_batch_stats(batch):
+    stats_dict = {
+        'records': len(batch),
+        'residues': sum(len(R.sequence) for R in batch),
+    }
+
+    return stats_dict
+
+
+def _args_dict_to_args_list(new_args):
+    """Converts args dict to list of arguments for Biolib"""
+
+    nested_list = [[f'--{key}', f'{value}'] for key, value in new_args.items()]
+
+    arg_list = []
+    for lst in nested_list:
+        for item in lst:
+            arg_list.append(item)
+
+    return arg_list

biolib/_runtime/runtime.py

@@ -0,0 +1,157 @@
+import json
+import re
+from pathlib import Path
+from typing import Optional
+
+from biolib import api
+from biolib._internal.runtime import BioLibRuntimeError, BioLibRuntimeNotRecognizedError, RuntimeJobDataDict
+from biolib.biolib_logging import logger
+from biolib.typing_utils import cast
+from biolib.utils.seq_util import SeqUtil
+
+
+class Runtime:
+    _job_data: Optional[RuntimeJobDataDict] = None
+
+    @staticmethod
+    def check_is_environment_biolib_app() -> bool:
+        return bool(Runtime._try_to_get_job_data())
+
+    @staticmethod
+    def check_is_environment_biolib_cloud() -> bool:
+        return Runtime._get_job_data().get('is_environment_biolib_cloud', False)
+
+    @staticmethod
+    def get_job_id() -> str:
+        return Runtime._get_job_data()['job_uuid']
+
+    @staticmethod
+    def get_job_auth_token() -> str:
+        return Runtime._get_job_data()['job_auth_token']
+
+    @staticmethod
+    def get_job_requested_machine() -> Optional[str]:
+        job_data = Runtime._get_job_data()
+        job_requested_machine = job_data.get('job_requested_machine')
+        if not job_requested_machine:
+            return None
+        return job_requested_machine
+
+    @staticmethod
+    def is_spot_machine_requested() -> bool:
+        job_data = Runtime._get_job_data()
+        return job_data.get('job_requested_machine_spot', False)
+
+    @staticmethod
+    def get_app_uri() -> str:
+        return Runtime._get_job_data()['app_uri']
+
+    @staticmethod
+    def get_max_workers() -> int:
+        return Runtime._get_job_data()['job_reserved_machines']
+
+    @staticmethod
+    def get_secret(secret_name: str) -> bytes:
+        assert re.match(
+            '^[a-zA-Z0-9_-]*$', secret_name
+        ), 'Secret name can only contain alphanumeric characters and dashes or underscores '
+        try:
+            with open(f'/biolib/secrets/{secret_name}', 'rb') as file:
+                return file.read()
+        except BaseException as error:
+            raise BioLibRuntimeError(f'Unable to get system secret: {secret_name}') from error
+
+    @staticmethod
+    def get_temporary_client_secret(secret_name: str) -> bytes:
+        assert re.match(
+            '^[a-zA-Z0-9_-]*$', secret_name
+        ), 'Secret name can only contain alphanumeric characters and dashes or underscores '
+        try:
+            with open(f'/biolib/temporary-client-secrets/{secret_name}', 'rb') as file:
+                return file.read()
+        except BaseException as error:
+            raise BioLibRuntimeError(f'Unable to get secret: {secret_name}') from error
+
+    @staticmethod
+    def set_main_result_prefix(result_prefix: str) -> None:
+        job_data = Runtime._get_job_data()
+        api.client.patch(
+            data={'result_name_prefix': result_prefix},
+            headers={'Job-Auth-Token': job_data['job_auth_token']},
+            path=f"/jobs/{job_data['job_uuid']}/main_result/",
+        )
+
+    @staticmethod
+    def set_result_name_prefix(result_prefix: str) -> None:
+        Runtime.set_main_result_prefix(result_prefix)
+
+    @staticmethod
+    def set_result_name_prefix_from_fasta(path_to_fasta: str) -> None:
+        """
+        Set BioLib result name to FASTA file name (excluding file extension) or,
+        if detecting a BioLib generated name, use ID of the first sequence in the fasta file
+        """
+        try:
+            # Set job name to first header if FASTA text input (random BioLib file) detected
+            if re.match('^input_[A-Za-z0-9]+.*', path_to_fasta):
+                first_id = next(SeqUtil.parse_fasta(path_to_fasta)).id
+                result_name = first_id.replace(' ', '_')[:60]
+            else:
+                result_name = Path(path_to_fasta).name
+
+            logger.debug(f'Setting result name to "{result_name}" from {path_to_fasta}')
+            Runtime.set_result_name_prefix(result_name)
+        except Exception as e:
+            logger.warning(f'Failed to set result name from fasta file {path_to_fasta}: {str(e)}')
+
+    @staticmethod
+    def set_result_name_from_file(path_to_file: str) -> None:
+        try:
+            if path_to_file.lower().endswith('.fasta'):
+                return Runtime.set_result_name_prefix_from_fasta(path_to_file)
+
+            # Set job name only if not a BioLib default name
+            if not re.match('^input_[A-Za-z0-9]+.*', path_to_file):
+                result_name = Path(path_to_file).name[:60]
+                logger.debug(f'Setting result name to "{result_name}" from {path_to_file}')
+                Runtime.set_result_name_prefix(result_name)
+        except Exception as e:
+            logger.warning(f'Failed to set result name from file {path_to_file}: {str(e)}')
+
+    @staticmethod
+    def set_result_name_from_string(result_name: str) -> None:
+        try:
+            truncated_name = result_name[:60]
+            logger.debug(f'Setting result name to "{truncated_name}" from string')
+            Runtime.set_result_name_prefix(truncated_name)
+        except Exception as e:
+            logger.warning(f'Failed to set result name from string: {str(e)}')
+
+    @staticmethod
+    def create_result_note(note: str) -> None:
+        job_id = Runtime.get_job_id()
+        # Note: Authentication is added by app caller proxy in compute node
+        api.client.post(data={'note': note}, path=f'/jobs/{job_id}/notes/')
+
+    @staticmethod
+    def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
+        if not Runtime._job_data:
+            try:
+                with open('/biolib/secrets/biolib_system_secret') as file:
+                    job_data: RuntimeJobDataDict = json.load(file)
+            except BaseException:
+                return None
+
+            if not job_data['version'].startswith('1.'):
+                raise BioLibRuntimeError(f"Unexpected system secret version {job_data['version']} expected 1.x.x")
+
+            Runtime._job_data = job_data
+
+        return cast(RuntimeJobDataDict, Runtime._job_data)
+
+    @staticmethod
+    def _get_job_data() -> RuntimeJobDataDict:
+        job_data = Runtime._try_to_get_job_data()
+        if not job_data:
+            raise BioLibRuntimeNotRecognizedError() from None
+        return job_data

biolib/_session/session.py

@@ -0,0 +1,44 @@
+from biolib import utils
+from biolib.typing_utils import Optional
+from biolib.api.client import ApiClient, ApiClientInitDict
+from biolib.app import BioLibApp
+
+
+class Session:
+    def __init__(self, _init_dict: ApiClientInitDict, _experiment: Optional[str] = None) -> None:
+        self._api = ApiClient(_init_dict=_init_dict)
+        self._experiment = _experiment
+
+    @staticmethod
+    def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None, experiment: Optional[str] = None) -> 'Session':
+        return Session(
+            _init_dict=ApiClientInitDict(
+                refresh_token=refresh_token,
+                base_url=base_url or utils.load_base_url_from_env(),
+                client_type=client_type,
+            ),
+            _experiment=experiment,
+        )
+
+    def load(self, uri: str, suppress_version_warning: bool = False) -> BioLibApp:
+        r"""Load a BioLib application by its URI or website URL.
+
+        Args:
+            uri (str): The URI or website URL of the application to load. Can be either:
+                - App URI (e.g., 'biolib/myapp:1.0.0')
+                - Website URL (e.g., 'https://biolib.com/biolib/myapp/')
+            suppress_version_warning (bool): If True, don't print a warning when no version is specified.
+                Defaults to False.
+
+        Returns:
+            BioLibApp: The loaded application object
+
+        Example::
+
+            >>> # Load by URI
+            >>> app = biolib.load('biolib/myapp:1.0.0')
+            >>> # Load by website URL
+            >>> app = biolib.load('https://biolib.com/biolib/myapp/')
+            >>> result = app.cli('--help')
+        """
+        return BioLibApp(uri=uri, _api_client=self._api, suppress_version_warning=suppress_version_warning, _experiment=self._experiment)

biolib/_shared/types/__init__.py

@@ -0,0 +1,74 @@
+from .account import AccountDict, AccountDetailedDict
+from .account_member import AccountMemberDict
+from .app import AppDetailedDict, AppSlimDict
+from .data_record import (
+    DataRecordDetailedDict,
+    DataRecordSlimDict,
+    DataRecordTypeDict,
+    DataRecordValidationRuleDict,
+    SqliteV1Column,
+    SqliteV1DatabaseSchema,
+    SqliteV1ForeignKey,
+    SqliteV1Table,
+)
+from .experiment import (
+    DeprecatedExperimentDict,
+    ExperimentDetailedDict,
+    ExperimentDict,
+    ResultCounts,
+)
+from .file_node import FileNodeDict, FileZipMetadataDict, ZipFileNodeDict
+from .push import PushResponseDict
+from .resource import ResourceDetailedDict, ResourceDict, ResourceTypeLiteral, ResourceUriDict, SemanticVersionDict
+from .resource_deploy_key import ResourceDeployKeyDict, ResourceDeployKeyWithSecretDict
+from .resource_permission import ResourcePermissionDetailedDict, ResourcePermissionDict
+from .resource_version import (
+    ResourceVersionAssetsDict,
+    ResourceVersionDetailedDict,
+    ResourceVersionDict,
+)
+from .result import ResultDetailedDict, ResultDict
+from .typing import Optional
+from .user import EnterpriseSettingsDict, UserDetailedDict, UserDict
+
+__all__ = [
+    'AccountDetailedDict',
+    'AccountDict',
+    'AccountMemberDict',
+    'AppDetailedDict',
+    'AppSlimDict',
+    'DataRecordDetailedDict',
+    'DataRecordSlimDict',
+    'DataRecordTypeDict',
+    'DataRecordValidationRuleDict',
+    'DeprecatedExperimentDict',
+    'EnterpriseSettingsDict',
+    'ExperimentDetailedDict',
+    'ExperimentDict',
+    'FileNodeDict',
+    'FileZipMetadataDict',
+    'Optional',
+    'PushResponseDict',
+    'ResourceDeployKeyDict',
+    'ResourceDeployKeyWithSecretDict',
+    'ResourceDetailedDict',
+    'ResourceDict',
+    'ResourceTypeLiteral',
+    'ResourcePermissionDetailedDict',
+    'ResourcePermissionDict',
+    'ResourceUriDict',
+    'ResourceVersionAssetsDict',
+    'ResourceVersionDetailedDict',
+    'ResourceVersionDict',
+    'ResultCounts',
+    'ResultDetailedDict',
+    'ResultDict',
+    'SemanticVersionDict',
+    'SqliteV1Column',
+    'SqliteV1DatabaseSchema',
+    'SqliteV1ForeignKey',
+    'SqliteV1Table',
+    'UserDetailedDict',
+    'UserDict',
+    'ZipFileNodeDict',
+]

biolib/_shared/types/account.py

@@ -0,0 +1,12 @@
+from .typing import TypedDict
+
+
+class AccountDict(TypedDict):
+    uuid: str
+    handle: str
+    display_name: str
+    description: str
+
+
+class AccountDetailedDict(AccountDict):
+    bio: str

biolib/_shared/types/account_member.py

@@ -0,0 +1,8 @@
+from .typing import Literal, TypedDict
+from .user import UserDict
+
+
+class AccountMemberDict(TypedDict):
+    user: UserDict
+    role: Literal['member', 'admin']
+    added_at: str

biolib/_shared/types/app.py

@@ -0,0 +1,9 @@
+from .typing import TypedDict
+
+
+class AppSlimDict(TypedDict):
+    pass
+
+
+class AppDetailedDict(AppSlimDict):
+    pass

biolib/_shared/types/data_record.py

@@ -0,0 +1,40 @@
+from .typing import Dict, List, Literal, Optional, TypedDict, Union
+
+
+class SqliteV1ForeignKey(TypedDict):
+    table: str
+    column: str
+
+
+class SqliteV1Column(TypedDict):
+    type: Literal['INTEGER', 'REAL', 'TEXT', 'JSON']
+    nullable: Optional[bool]
+    foreign_key: Optional[SqliteV1ForeignKey]
+    json_schema: Optional[Dict]
+
+
+class SqliteV1Table(TypedDict):
+    columns: Dict[str, SqliteV1Column]
+
+
+class SqliteV1DatabaseSchema(TypedDict):
+    tables: Dict[str, SqliteV1Table]
+
+
+class DataRecordValidationRuleDict(TypedDict):
+    path: str
+    type: str
+    rule: Union[SqliteV1DatabaseSchema]
+
+
+class DataRecordTypeDict(TypedDict):
+    name: str
+    validation_rules: List[DataRecordValidationRuleDict]
+
+
+class DataRecordSlimDict(TypedDict):
+    pass
+
+
+class DataRecordDetailedDict(DataRecordSlimDict):
+    type: Optional[DataRecordTypeDict]

biolib/_shared/types/experiment.py

@@ -0,0 +1,32 @@
+from .result import ResultDict
+from .typing import Optional, TypedDict
+
+
+class ResultCounts(TypedDict):
+    cancelled: int
+    completed: int
+    failed: int
+    in_progress: int
+    queued: int
+    total: int
+
+
+class DeprecatedExperimentDict(TypedDict):
+    # Note: fields on this TypedDict are deprecated
+    job_count: int
+    job_running_count: int
+
+
+class ExperimentDict(DeprecatedExperimentDict):
+    uuid: Optional[str]
+    name: Optional[str]
+    account_uuid: Optional[str]
+    created_at: Optional[str]
+    finished_at: Optional[str]
+    last_created_at: Optional[str]
+    last_created_result: Optional[ResultDict]
+    result_counts: ResultCounts
+
+
+class ExperimentDetailedDict(ExperimentDict):
+    pass