pybiolib 1.2.1056__py3-none-any.whl → 1.2.1642__py3-none-any.whl

biolib/jobs/job.py

@@ -2,7 +2,7 @@ import base64
 import sys
 import time
 from collections import OrderedDict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from urllib.parse import urlparse
 
@@ -10,7 +10,8 @@ import biolib.api.client
 from biolib import utils
 from biolib._internal.http_client import HttpClient
 from biolib._internal.tree_utils import build_tree_from_files, build_tree_str
-from biolib._internal.utils import open_browser_window_from_notebook
+from biolib._internal.utils import PathFilter, filter_lazy_loaded_files, open_browser_window_from_notebook
+from biolib._shared.utils import parse_resource_uri
 from biolib.api.client import ApiClient
 from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
@@ -18,16 +19,16 @@ from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import LazyLoadedFile, ModuleInput, ModuleInputDict, ModuleOutputV2
 from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
 from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
+from biolib.biolib_binary_format.utils import InMemoryIndexableBuffer
 from biolib.biolib_errors import BioLibError, CloudJobFinishedError
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node.job_worker.job_storage import JobStorage
 from biolib.compute_node.utils import SystemExceptionCodeMap, SystemExceptionCodes
-from biolib.jobs.job_result import JobResult, PathFilter
+from biolib.jobs.job_result import JobResult
 from biolib.jobs.types import CloudJobDict, CloudJobStartedDict, JobDict
 from biolib.tables import BioLibTable
-from biolib.typing_utils import Dict, List, Optional, Union, cast
+from biolib.typing_utils import Dict, Generator, List, Optional, Tuple, Union, cast
 from biolib.utils import IS_RUNNING_IN_NOTEBOOK
-from biolib.utils.app_uri import parse_app_uri
 
 
 class Result:
@@ -49,7 +50,7 @@ class Result:
         self._auth_token: str = job_dict['auth_token']
 
         self._job_dict: JobDict = job_dict
-        self._job_dict_last_fetched_at: datetime = datetime.utcnow()
+        self._job_dict_last_fetched_at: datetime = datetime.now(timezone.utc)
         self._result: Optional[JobResult] = None
         self._cached_input_arguments: Optional[List[str]] = None
 
@@ -187,6 +188,45 @@ class Result:
         """
         return self.result.list_output_files(path_filter=path_filter)
 
+    def list_input_files(
+        self,
+        path_filter: Optional[PathFilter] = None,
+    ) -> List[LazyLoadedFile]:
+        """List input files from the result.
+
+        Args:
+            path_filter (PathFilter, optional): Filter to apply to the input files.
+                Can be a string glob pattern or a callable that takes a path string and returns a boolean.
+
+        Returns:
+            List[LazyLoadedFile]: List of input files.
+
+        Example::
+            >>> result = biolib.get_result("result_id")
+            >>> input_files = result.list_input_files()
+            >>> # Filter files with a glob pattern
+            >>> input_files = result.list_input_files("*.txt")
+        """
+        presigned_download_url = BiolibJobApi.get_job_storage_download_url(
+            job_uuid=self.id,
+            job_auth_token=self._auth_token,
+            storage_type='input',
+        )
+        response = HttpClient.request(url=presigned_download_url)
+        module_input_serialized: bytes = response.content
+        module_input = ModuleInput(module_input_serialized).deserialize()
+
+        files = []
+        for path, data in module_input['files'].items():
+            buffer = InMemoryIndexableBuffer(data)
+            lazy_file = LazyLoadedFile(path=path, buffer=buffer, start=0, length=len(data))
+            files.append(lazy_file)
+
+        if not path_filter:
+            return files
+
+        return filter_lazy_loaded_files(files, path_filter)
+
     def get_output_file(self, filename: str) -> LazyLoadedFile:
         return self.result.get_output_file(filename=filename)
 
@@ -256,14 +296,44 @@ class Result:
         self,
         output_dir: str,
         path_filter: Optional[PathFilter] = None,
-        skip_file_if_exists: Optional[bool] = None,
+        skip_file_if_exists: bool = False,
         overwrite: bool = False,
+        flat: bool = False,
     ) -> None:
+        """Save output files from the result to a local directory.
+
+        Args:
+            output_dir (str): Directory path where files will be saved.
+            path_filter (PathFilter, optional): Filter to apply to output files.
+                Can be a string glob pattern or a callable that takes a path and returns a boolean.
+            skip_file_if_exists (bool, optional): If True, skip files that already exist locally.
+                Defaults to False.
+            overwrite (bool, optional): If True, overwrite existing files by renaming them with a timestamp.
+                Defaults to False.
+            flat (bool, optional): If True, save all files directly to output_dir using only their basenames,
+                without creating subdirectories. When enabled, raises an error if duplicate basenames exist
+                in the filtered output or if any basename already exists in output_dir. Defaults to False.
+
+        Raises:
+            BioLibError: If flat=True and duplicate basenames are found in filtered output.
+            BioLibError: If flat=True and a file with the same basename already exists in output_dir.
+            BioLibError: If a file already exists and neither skip_file_if_exists nor overwrite is True.
+
+        Example::
+            >>> result = biolib.get_result("result_id")
+            >>> # Save all files preserving directory structure
+            >>> result.save_files("./output")
+            >>> # Save files flat without subdirectories
+            >>> result.save_files("./output", flat=True)
+            >>> # Save only specific files
+            >>> result.save_files("./output", path_filter="*.txt")
+        """
         self.result.save_files(
             output_dir=output_dir,
             path_filter=path_filter,
             skip_file_if_exists=skip_file_if_exists,
             overwrite=overwrite,
+            flat=flat,
         )
 
     def get_status(self) -> str:
@@ -358,6 +428,7 @@ class Result:
             >>> # Recompute with different arguments
             >>> new_result = result.recompute(arguments=["--new-arg", "value"])
         """
+        self._refetch_job_dict()
         app_response = BiolibAppApi.get_by_uri(uri=app_uri or self._job_dict['app_uri'])
 
         job_storage_input = RemoteJobStorageEndpoint(
@@ -385,6 +456,7 @@ class Result:
             app_uri=app_response['app_uri'],
             app_version_uuid=app_response['app_version']['public_id'],
             module_input_serialized=module_input_serialized,
+            override_command=self._job_dict['arguments_override_command'],
             machine=machine if machine else original_requested_machine,
         )
         if blocking:
@@ -451,16 +523,11 @@ class Result:
         return Result(job_dict)
 
     @staticmethod
-    def print_logs_packages(stdout_and_stderr_packages_b64):
+    def _yield_logs_packages(stdout_and_stderr_packages_b64) -> Generator[Tuple[str, bytes], None, None]:
         for stdout_and_stderr_package_b64 in stdout_and_stderr_packages_b64:
             stdout_and_stderr_package = base64.b64decode(stdout_and_stderr_package_b64)
             stdout_and_stderr = StdoutAndStderr(stdout_and_stderr_package).deserialize()
-
-            sys.stdout.write(stdout_and_stderr.decode())
-            if not IS_RUNNING_IN_NOTEBOOK:  # for some reason flushing in jupyter notebooks breaks \r handling
-                sys.stdout.flush()
-        # flush after having processed all packages
-        sys.stdout.flush()
+            yield ('stdout', stdout_and_stderr)
 
     def show(self) -> None:
         self._refetch_job_dict()
@@ -470,21 +537,44 @@ class Result:
             title=f'Result: {self._uuid}',
         ).print_table()
 
-    def stream_logs(self) -> None:
+    def stream_logs(self, as_iterator: bool = False):
+        if as_iterator:
+            return self._iter_logs()
         self._stream_logs()
+        return None
 
     def _stream_logs(self, enable_print: bool = True) -> None:
+        try:
+            for stream_type, data in self._iter_logs(enable_print=enable_print):
+                if stream_type == 'stdout':
+                    if IS_RUNNING_IN_NOTEBOOK:
+                        sys.stdout.write(data.decode(encoding='utf-8', errors='replace'))
+                        # Note: we avoid flush() in notebook as that breaks \r handling
+                    else:
+                        sys.stdout.buffer.write(data)
+                        sys.stdout.buffer.flush()
+                elif stream_type == 'stderr':
+                    if IS_RUNNING_IN_NOTEBOOK:
+                        sys.stderr.write(data.decode(encoding='utf-8', errors='replace'))
+                        # Note: we avoid flush() in notebook as that breaks \r handling
+                    else:
+                        sys.stderr.buffer.write(data)
+                        sys.stderr.buffer.flush()
+        finally:
+            # Flush after having processed all packages
+            if IS_RUNNING_IN_NOTEBOOK:
+                sys.stdout.flush()
+                sys.stderr.flush()
+
+    def _iter_logs(self, enable_print: bool = True) -> Generator[Tuple[str, bytes], None, None]:
         try:
             cloud_job = self._get_cloud_job_awaiting_started()
         except CloudJobFinishedError:
             logger.info(f'--- The result {self.id} has already completed (no streaming will take place) ---')
             logger.info('--- The stdout log is printed below: ---')
-            sys.stdout.flush()
-            print(self.get_stdout().decode(), file=sys.stdout)
-            sys.stdout.flush()
+            yield ('stdout', self.get_stdout())
             logger.info('--- The stderr log is printed below: ---')
-            print(self.get_stderr().decode(), file=sys.stderr)
-            sys.stderr.flush()
+            yield ('stderr', self.get_stderr())
             logger.info(f'--- The job {self.id} has already completed. Its output was printed above. ---')
             return
 
@@ -496,7 +586,7 @@ class Result:
             logger_no_user_data.debug(f'Using cloud proxy URL from env var BIOLIB_CLOUD_BASE_URL: {compute_node_url}')
 
         if enable_print:
-            self._print_full_logs(node_url=compute_node_url)
+            yield from self._yield_full_logs(node_url=compute_node_url)
 
         final_status_messages: List[str] = []
         while True:
@@ -515,8 +605,8 @@ class Result:
                     # Print the status before writing stdout and stderr
                     logger.info(f'Cloud: {status_update["log_message"]}')
 
-            if 'stdout_and_stderr_packages_b64' and enable_print:
-                self.print_logs_packages(status_json['stdout_and_stderr_packages_b64'])
+            if enable_print:
+                yield from self._yield_logs_packages(status_json['stdout_and_stderr_packages_b64'])
 
             if 'error_code' in status_json:
                 error_code = status_json['error_code']
@@ -533,7 +623,7 @@ class Result:
 
         self.wait()  # Wait for compute node to tell the backend that the job is finished
 
-    def _print_full_logs(self, node_url: str) -> None:
+    def _yield_full_logs(self, node_url: str) -> Generator[Tuple[str, bytes], None, None]:
         try:
             response_json = HttpClient.request(url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full').json()
         except Exception as error:
@@ -543,7 +633,7 @@ class Result:
         for status_update in response_json.get('previous_status_updates', []):
             logger.info(f'Cloud: {status_update["log_message"]}')
 
-        self.print_logs_packages(response_json['streamed_logs_packages_b64'])
+        yield from self._yield_logs_packages(response_json['streamed_logs_packages_b64'])
 
     def _get_cloud_job_awaiting_started(self) -> CloudJobStartedDict:
         retry_count = 0
@@ -589,11 +679,11 @@ class Result:
         )
 
     def _refetch_job_dict(self, force_refetch: Optional[bool] = False) -> None:
-        if not force_refetch and self._job_dict_last_fetched_at > datetime.utcnow() - timedelta(seconds=2):
+        if not force_refetch and self._job_dict_last_fetched_at > datetime.now(timezone.utc) - timedelta(seconds=2):
             return
 
         self._job_dict = self._get_job_dict(self._uuid, self._auth_token)
-        self._job_dict_last_fetched_at = datetime.utcnow()
+        self._job_dict_last_fetched_at = datetime.now(timezone.utc)
 
     @staticmethod
     def _start_job_in_cloud(
@@ -612,8 +702,8 @@ class Result:
     ) -> 'Result':
         if len(module_input_serialized) < 500_000 and temporary_client_secrets is None:
             _job_dict = BiolibJobApi.create_job_with_data(
-                app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
                 app_version_uuid=app_version_uuid,
+                app_resource_name_prefix=parse_resource_uri(app_uri)['resource_prefix'],
                 arguments_override_command=override_command,
                 experiment_uuid=experiment_id,
                 module_input_serialized=module_input_serialized,
@@ -627,8 +717,8 @@ class Result:
             return Result(cast(JobDict, _job_dict))
 
         job_dict: CreatedJobDict = BiolibJobApi.create(
-            app_resource_name_prefix=parse_app_uri(app_uri)['resource_name_prefix'],
             app_version_id=app_version_uuid,
+            app_resource_name_prefix=parse_resource_uri(app_uri)['resource_prefix'],
             experiment_uuid=experiment_id,
             machine=machine,
             notify=notify,

biolib/jobs/job_result.py

@@ -1,16 +1,14 @@
 import time
-from fnmatch import fnmatch
 from pathlib import Path
 
+from biolib._internal.utils import PathFilter, filter_lazy_loaded_files
 from biolib.biolib_binary_format import ModuleOutputV2
 from biolib.biolib_binary_format.remote_endpoints import RemoteJobStorageEndpoint
 from biolib.biolib_binary_format.remote_stream_seeker import StreamSeeker
 from biolib.biolib_binary_format.utils import LazyLoadedFile, RemoteIndexableBuffer
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
-from biolib.typing_utils import Callable, List, Optional, Union, cast
-
-PathFilter = Union[str, Callable[[str], bool]]
+from biolib.typing_utils import Dict, List, Optional
 
 
 class JobResult:
@@ -38,18 +36,68 @@ class JobResult:
         self,
         output_dir: str,
         path_filter: Optional[PathFilter] = None,
-        skip_file_if_exists: Optional[bool] = None,
+        skip_file_if_exists: bool = False,
         overwrite: bool = False,
+        flat: bool = False,
     ) -> None:
         module_output = self._get_module_output()
         output_files = module_output.get_files()
-        filtered_output_files = self._get_filtered_files(output_files, path_filter) if path_filter else output_files
+        filtered_output_files = filter_lazy_loaded_files(output_files, path_filter) if path_filter else output_files
 
         if len(filtered_output_files) == 0:
             logger.debug('No output files to save')
             return
 
-        total_files_data_to_download_in_bytes = sum(file.length for file in filtered_output_files)
+        if flat:
+            basename_to_paths: Dict[str, List[str]] = {}
+            for file in filtered_output_files:
+                basename = Path(file.path).name
+                if basename not in basename_to_paths:
+                    basename_to_paths[basename] = []
+                basename_to_paths[basename].append(file.path)
+
+            duplicates = {basename: paths for basename, paths in basename_to_paths.items() if len(paths) > 1}
+
+            if duplicates:
+                max_shown = 3
+                error_parts = []
+                sorted_basenames = sorted(duplicates.keys())
+
+                for basename in sorted_basenames[:max_shown]:
+                    paths = duplicates[basename]
+                    error_parts.append(f'  {basename}: ({", ".join(paths)})')
+
+                error_message = 'Cannot save files in flat mode: duplicate filenames detected:\n' + '\n'.join(
+                    error_parts
+                )
+
+                if len(duplicates) > max_shown:
+                    remaining = len(duplicates) - max_shown
+                    error_message += f'\n  (and {remaining} more)'
+
+                raise BioLibError(error_message)
+
+        major_gap_threshold = 50_000
+        n = len(filtered_output_files)
+
+        next_break_end = [0] * n
+        if n > 0:
+            next_break_end[n - 1] = filtered_output_files[n - 1].start + filtered_output_files[n - 1].length
+            for i in range(n - 2, -1, -1):
+                end_i = filtered_output_files[i].start + filtered_output_files[i].length
+                gap = filtered_output_files[i + 1].start - end_i
+                if gap >= major_gap_threshold:
+                    next_break_end[i] = end_i
+                else:
+                    next_break_end[i] = next_break_end[i + 1]
+
+        total_files_data_to_download_in_bytes = 0
+        file_read_ahead_map = {}
+        for i, file in enumerate(filtered_output_files):
+            total_files_data_to_download_in_bytes += file.length
+            end_i = file.start + file.length
+            read_ahead_bytes = max(0, next_break_end[i] - end_i)
+            file_read_ahead_map[i] = read_ahead_bytes
 
         # Assume files are in order
         first_file = filtered_output_files[0]
@@ -57,14 +105,18 @@ class JobResult:
         stream_seeker = StreamSeeker(
             files_data_start=first_file.start,
             files_data_end=last_file.start + last_file.length,
-            download_chunk_size_in_bytes=min(total_files_data_to_download_in_bytes, 10_000_000),
+            max_chunk_size=min(total_files_data_to_download_in_bytes, 10_000_000),
             upstream_buffer=module_output.buffer,
         )
 
         logger.info(f'Saving {len(filtered_output_files)} files to {output_dir}...')
-        for file in filtered_output_files:
-            # Remove leading slash of file_path
-            destination_file_path = Path(output_dir) / Path(file.path.lstrip('/'))
+        for file_index, file in enumerate(filtered_output_files):
+            if flat:
+                destination_file_path = Path(output_dir) / Path(file.path).name
+            else:
+                # Remove leading slash of file_path
+                destination_file_path = Path(output_dir) / Path(file.path.lstrip('/'))
+
             if destination_file_path.exists():
                 if skip_file_if_exists:
                     print(f'Skipping {destination_file_path} as a file with that name already exists locally.')
@@ -91,8 +143,12 @@ class JobResult:
                 file_start += data_already_downloaded
                 data_to_download -= data_already_downloaded
 
+            read_ahead_bytes = file_read_ahead_map[file_index]
+
             with open(partial_path, mode='ab') as partial_file:
-                for chunk in stream_seeker.seek_and_read(file_start=file_start, file_length=data_to_download):
+                for chunk in stream_seeker.seek_and_read(
+                    file_start=file_start, file_length=data_to_download, read_ahead_bytes=read_ahead_bytes
+                ):
                     partial_file.write(chunk)
 
             # rename partial file to actual file name
@@ -100,7 +156,7 @@ class JobResult:
 
     def get_output_file(self, filename) -> LazyLoadedFile:
         files = self._get_module_output().get_files()
-        filtered_files = self._get_filtered_files(files, path_filter=filename)
+        filtered_files = filter_lazy_loaded_files(files, path_filter=filename)
         if not filtered_files:
             raise BioLibError(f'File {filename} not found in results.')
 
@@ -114,26 +170,7 @@ class JobResult:
         if not path_filter:
             return files
 
-        return self._get_filtered_files(files, path_filter)
-
-    @staticmethod
-    def _get_filtered_files(files: List[LazyLoadedFile], path_filter: PathFilter) -> List[LazyLoadedFile]:
-        if not (isinstance(path_filter, str) or callable(path_filter)):
-            raise Exception('Expected path_filter to be a string or a function')
-
-        if callable(path_filter):
-            return list(filter(lambda x: path_filter(x.path), files))  # type: ignore
-
-        glob_filter = cast(str, path_filter)
-
-        # since all file paths start with /, make sure filter does too
-        if not glob_filter.startswith('/'):
-            glob_filter = '/' + glob_filter
-
-        def _filter_function(file: LazyLoadedFile) -> bool:
-            return fnmatch(file.path, glob_filter)
-
-        return list(filter(_filter_function, files))
+        return filter_lazy_loaded_files(files, path_filter)
 
     def _get_module_output(self) -> ModuleOutputV2:
         if self._module_output is None:

biolib/jobs/types.py

@@ -26,6 +26,7 @@ class Result(TypedDict):
 
 class JobDict(TypedDict):
     app_uri: str
+    arguments_override_command: bool
     auth_token: str
     created_at: str
     ended_at: Optional[str]

biolib/sdk/__init__.py

@@ -2,6 +2,7 @@ from typing import Optional
 
 # Imports to hide and use as private internal utils
 from biolib._data_record.data_record import DataRecord as _DataRecord
+from biolib._index.index import Index as _Index
 from biolib._internal.push_application import push_application as _push_application
 from biolib._internal.push_application import set_app_version_as_active as _set_app_version_as_active
 from biolib._runtime.runtime import Runtime as _Runtime
@@ -12,8 +13,18 @@ from biolib.app import BioLibApp as _BioLibApp
 Runtime = _Runtime
 
 
-def get_session(refresh_token: str, base_url: Optional[str] = None, client_type: Optional[str] = None) -> _Session:
-    return _Session.get_session(refresh_token=refresh_token, base_url=base_url, client_type=client_type)
+def get_session(
+    refresh_token: str,
+    base_url: Optional[str] = None,
+    client_type: Optional[str] = None,
+    experiment: Optional[str] = None,
+) -> _Session:
+    return _Session.get_session(
+        refresh_token=refresh_token,
+        base_url=base_url,
+        client_type=client_type,
+        experiment=experiment,
+    )
 
 
 def push_app_version(uri: str, path: str) -> _BioLibApp:
@@ -64,3 +75,7 @@ def create_data_record(
         data_path=data_path,
         record_type=record_type,
     )
+
+
+def get_index(uri: str) -> _Index:
+    return _Index.get_by_uri(uri)

biolib/typing_utils.py

@@ -1,2 +1,2 @@
 # TODO: Deprecate and later remove this file
-from biolib._internal.types.typing import *  # pylint: disable=wildcard-import, unused-wildcard-import
+from biolib._shared.types.typing import *  # pylint: disable=wildcard-import, unused-wildcard-import

biolib/utils/cache_state.py

@@ -2,7 +2,7 @@ import os
 import abc
 import json
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 
 import appdirs  # type: ignore
 
@@ -88,4 +88,4 @@ class CacheState(abc.ABC, Generic[StateType]):
 
     @staticmethod
     def get_timestamp_now() -> str:
-        return datetime.utcnow().isoformat()
+        return datetime.now(timezone.utc).isoformat()

biolib/utils/seq_util.py

@@ -9,7 +9,7 @@ class SeqUtilRecord:
         self,
         sequence: str,
         sequence_id: str,
-        description: Optional['str'],
+        description: Optional['str'] = None,
         properties: Optional[Dict[str, str]] = None,
     ):
         self.sequence = sequence

{pybiolib-1.2.1056.dist-info → pybiolib-1.2.1642.dist-info}/METADATA

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: pybiolib
-Version: 1.2.1056
+Version: 1.2.1642
 Summary: BioLib Python Client
 License: MIT
+License-File: LICENSE
 Keywords: biolib
 Author: biolib
 Author-email: hello@biolib.com
@@ -17,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: compute-node
 Requires-Dist: appdirs (>=1.4.3)
 Requires-Dist: click (>=8.0.0)