pybiolib 1.2.1056__py3-none-any.whl → 1.2.1727__py3-none-any.whl

biolib/__init__.py

@@ -1,3 +1,4 @@
+# ruff: noqa: I001
 # Imports to hide
 import os
 from urllib.parse import urlparse as _urlparse
@@ -15,6 +16,7 @@ from biolib.jobs.job import Result as _Result
 from biolib import user as _user
 from biolib.typing_utils import List, Optional, cast as _cast
 from biolib._data_record.data_record import DataRecord as _DataRecord
+from biolib._internal.utils.job_url import parse_result_id_or_url as _parse_result_id_or_url
 
 import biolib.api
 import biolib.app
@@ -22,7 +24,6 @@ import biolib.cli
 import biolib.sdk
 import biolib.utils
 
-
 # ------------------------------------ Function definitions for public Python API ------------------------------------
 
 
@@ -83,43 +84,65 @@ def search(
 
 
 def get_job(job_id: str, job_token: Optional[str] = None) -> _Result:
-    r"""Get a job by its ID.
+    r"""Get a job by its ID or full URL.
 
     Args:
-        job_id (str): The UUID of the job to retrieve
+        job_id (str): The UUID of the job to retrieve, or a full URL to the job.
+            Can be either:
+            - Job UUID (e.g., 'abc123')
+            - Full URL (e.g., 'https://biolib.com/result/abc123/?token=xyz789')
+            - Full URL with token parameter (e.g., 'biolib.com/result/abc123/token=xyz789')
         job_token (str, optional): Authentication token for accessing the job.
             Only needed for jobs that aren't owned by the current user.
+            If the URL contains a token, this parameter is ignored.
 
     Returns:
         Job: The job object
 
     Example::
 
+        >>> # Get by UUID
         >>> job = biolib.get_job('abc123')
-        >>> # Access shared job
+        >>> # Get with explicit token
         >>> job = biolib.get_job('abc123', job_token='xyz789')
+        >>> # Get by full URL with token
+        >>> job = biolib.get_job('https://biolib.com/result/abc123/?token=xyz789')
+        >>> # Get by URL with inline token format
+        >>> job = biolib.get_job('biolib.com/result/abc123/token=xyz789')
     """
-    return _Result.create_from_uuid(uuid=job_id, auth_token=job_token)
+    uuid, token = _parse_result_id_or_url(job_id, job_token)
+    return _Result.create_from_uuid(uuid=uuid, auth_token=token)
 
 
 def get_result(result_id: str, result_token: Optional[str] = None) -> _Result:
-    r"""Get a result by its ID.
+    r"""Get a result by its ID or full URL.
 
     Args:
-        result_id (str): The UUID of the result to retrieve
+        result_id (str): The UUID of the result to retrieve, or a full URL to the result.
+            Can be either:
+            - Result UUID (e.g., 'abc123')
+            - Full URL (e.g., 'https://biolib.com/result/abc123/?token=xyz789')
+            - Full URL with token parameter (e.g., 'biolib.com/result/abc123/token=xyz789')
         result_token (str, optional): Authentication token for accessing the result.
-            Only needed for result that aren't owned by the current user.
+            Only needed for results that aren't owned by the current user.
+            If the URL contains a token, this parameter is ignored.
 
     Returns:
         Result: The result object
 
     Example::
 
+        >>> # Get by UUID
         >>> result = biolib.get_result('abc123')
-        >>> # Access shared result
+        >>> # Get with explicit token
         >>> result = biolib.get_result('abc123', result_token='xyz789')
+        >>> # Get by full URL with token
+        >>> result = biolib.get_result('https://biolib.com/result/abc123/?token=xyz789')
+        >>> # Get by URL with inline token format
+        >>> result = biolib.get_result('biolib.com/result/abc123/token=xyz789')
     """
-    return _Result.create_from_uuid(uuid=result_id, auth_token=result_token)
+    uuid, token = _parse_result_id_or_url(result_id, result_token)
+    return _Result.create_from_uuid(uuid=uuid, auth_token=token)
 
 
 def get_data_record(uri: str) -> _DataRecord:

biolib/_data_record/data_record.py

@@ -3,26 +3,27 @@ from collections import namedtuple
 from datetime import datetime
 from pathlib import Path
 from struct import Struct
-from typing import Callable, Dict, Iterable, List, Optional, Union, cast
+from typing import Callable, Dict, Iterable, Iterator, List, Optional, Union, cast
 
 from biolib import api
-from biolib._internal import types
 from biolib._internal.data_record import get_data_record_state_from_uri
 from biolib._internal.data_record.data_record import validate_sqlite_v1
 from biolib._internal.data_record.push_data import (
+    _upload_from_iterator,
     push_data_path,
     validate_data_path_and_get_files_and_size_of_directory,
 )
 from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
 from biolib._internal.http_client import HttpClient
-from biolib._internal.types.file_node import ZipFileNodeDict
+from biolib._shared import types
+from biolib._shared.types import ZipFileNodeDict
+from biolib._shared.utils import parse_resource_uri
 from biolib.api import client as api_client
 from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersion, DataRecordVersionInfo
+from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersionInfo
 from biolib.biolib_binary_format import LazyLoadedFile
 from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
 from biolib.biolib_logging import logger
-from biolib.utils.app_uri import parse_app_uri
 
 PathFilter = Union[str, List[str], Callable[[str], bool]]
 
@@ -44,11 +45,11 @@ class DataRecord:
 
     @property
     def name(self) -> str:
-        uri_parsed = parse_app_uri(self._state['resource_uri'], use_account_as_name_default=False)
-        if not uri_parsed['app_name']:
+        uri_parsed = parse_resource_uri(self._state['resource_uri'], use_account_as_name_default=False)
+        if not uri_parsed['resource_name']:
             raise ValueError('Expected parameter "resource_uri" to contain resource name')
 
-        return uri_parsed['app_name']
+        return uri_parsed['resource_name']
 
     def list_files(
         self,
@@ -113,37 +114,36 @@ class DataRecord:
                 else:
                     raise Exception(f"Error processing data record validation: unknown rule type {rule['type']}")
 
-        response = api.client.post(path='/lfs/versions/', data={'resource_uuid': self._state['resource_uuid']})
-        data_record_version: DataRecordVersion = response.json()
-        resource_version_uuid = data_record_version['uuid']
-
-        push_data_path(
+        new_resource_version_uuid = push_data_path(
             data_path=data_path,
             data_size_in_bytes=data_size_in_bytes,
             files_to_zip=files_to_zip,
-            resource_version_uuid=resource_version_uuid,
+            resource_uuid=self._state['resource_uuid'],
             chunk_size_in_mb=chunk_size_in_mb,
+            publish=True,
         )
 
-        api.client.patch(
-            path=f'/resources/versions/{resource_version_uuid}/',
-            data={'state': 'published', 'set_as_active': True},
-        )
-
-        logger.info(f"Successfully pushed a new Data Record version '{data_record_version['uri']}'")
-        self._state = get_data_record_state_from_uri(data_record_version['uri'])
+        updated_record = DataRecord._get_by_version_uuid(new_resource_version_uuid)
+        self._state = updated_record._state  # pylint: disable=protected-access
+        logger.info(f"Successfully pushed a new Data Record version '{self.uri}'")
 
     @staticmethod
     def get_by_uri(uri: str) -> 'DataRecord':
         return DataRecord(_internal_state=get_data_record_state_from_uri(uri))
 
+    @staticmethod
+    def _get_by_version_uuid(version_uuid: str) -> 'DataRecord':
+        response = api.client.get(path=f'/lfs/versions/{version_uuid}/')
+        version_info = response.json()
+        return DataRecord.get_by_uri(version_info['uri'])
+
     @staticmethod
     def create(destination: str, data_path: Optional[str] = None, record_type: Optional[str] = None) -> 'DataRecord':
         BiolibApiClient.assert_is_signed_in(authenticated_action_description='create a Data Record')
         if data_path is not None:
             assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
-        uri_parsed = parse_app_uri(destination, use_account_as_name_default=False)
-        if uri_parsed['app_name_normalized']:
+        uri_parsed = parse_resource_uri(destination, use_account_as_name_default=False)
+        if uri_parsed['resource_name_normalized']:
             data_record_uri = destination
         else:
             record_name = 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
@@ -173,10 +173,10 @@ class DataRecord:
             'resource_type': 'data-record',
         }
         if uri:
-            uri_parsed = parse_app_uri(uri, use_account_as_name_default=False)
+            uri_parsed = parse_resource_uri(uri, use_account_as_name_default=False)
             params['account_handle'] = uri_parsed['account_handle_normalized']
-            if uri_parsed['app_name_normalized']:
-                params['app_name'] = uri_parsed['app_name_normalized']
+            if uri_parsed['resource_name_normalized']:
+                params['app_name'] = uri_parsed['resource_name_normalized']
 
         results = api_client.get(path='/apps/', params=params).json()['results']
         if count is None and len(results) == max_page_size:
@@ -284,3 +284,80 @@ class DataRecord:
 
     def _get_detailed_dict(self) -> types.DataRecordDetailedDict:
         return cast(types.DataRecordDetailedDict, api_client.get(f'/resources/data-records/{self.uuid}/').json())
+
+    def _get_zip_size_bytes(self) -> int:
+        remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
+            resource_version_uuid=self._state['resource_version_uuid'],
+        )
+        presigned_url = remote_storage_endpoint.get_remote_url()
+        response = HttpClient.request(url=presigned_url, headers={'range': 'bytes=0-0'})
+        content_range = response.headers.get('Content-Range', '')
+        if not content_range or '/' not in content_range:
+            raise ValueError('Unable to determine zip size: Content-Range header missing or invalid')
+        total_size = int(content_range.split('/')[1])
+        return total_size
+
+    def _iter_zip_bytes(self, chunk_size_bytes: int) -> Iterator[bytes]:
+        remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
+            resource_version_uuid=self._state['resource_version_uuid'],
+        )
+        presigned_url = remote_storage_endpoint.get_remote_url()
+        response = HttpClient.request(url=presigned_url, headers={'range': 'bytes=0-0'})
+        content_range = response.headers.get('Content-Range', '')
+        if not content_range or '/' not in content_range:
+            raise ValueError('Unable to determine zip size: Content-Range header missing or invalid')
+        total_size = int(content_range.split('/')[1])
+
+        for start in range(0, total_size, chunk_size_bytes):
+            end = min(start + chunk_size_bytes - 1, total_size - 1)
+            presigned_url = remote_storage_endpoint.get_remote_url()
+            response = HttpClient.request(
+                url=presigned_url,
+                headers={'range': f'bytes={start}-{end}'},
+                timeout_in_seconds=300,
+            )
+            yield response.content
+
+    @staticmethod
+    def clone(
+        source: 'DataRecord',
+        destination: 'DataRecord',
+        on_progress: Optional[Callable[[int, int], None]] = None,
+    ) -> 'DataRecord':
+        BiolibApiClient.assert_is_signed_in(authenticated_action_description='clone a Data Record')
+
+        # pylint: disable=protected-access
+        total_size_in_bytes = source._get_zip_size_bytes()
+
+        if total_size_in_bytes == 0:
+            raise ValueError('Source data record has no data to clone')
+
+        min_chunk_size_bytes = 10_000_000
+        chunk_size_in_bytes = max(min_chunk_size_bytes, int(total_size_in_bytes / 9_000))
+
+        zip_iterator = source._iter_zip_bytes(chunk_size_bytes=chunk_size_in_bytes)
+
+        new_resource_version_uuid = _upload_from_iterator(
+            resource_uuid=destination._state['resource_uuid'],
+            payload_iterator=zip_iterator,
+            payload_size_in_bytes=total_size_in_bytes,
+            publish=True,
+            on_progress=on_progress,
+        )
+        # pylint: enable=protected-access
+
+        logger.info(f"Successfully cloned data to '{destination.uri}'")
+        return DataRecord._get_by_version_uuid(new_resource_version_uuid)
+
+    def delete(self) -> None:
+        """Delete the data record.
+
+        Example::
+            >>> record = DataRecord.get_by_uri("account/data-record")
+            >>> record.delete()
+        """
+        try:
+            api_client.delete(path=f'/apps/{self.uuid}/')
+            logger.info(f'Data record {self.uri} deleted')
+        except Exception as error:
+            raise Exception(f'Failed to delete data record {self.uri} due to: {error}') from error

biolib/_index/index.py

@@ -0,0 +1,51 @@
+import json
+from typing import Any, Dict
+
+from biolib import api
+from biolib._index.types import IndexInfo
+from biolib._internal.index import get_index_from_uri
+from biolib.biolib_api_client import BiolibApiClient
+from biolib.biolib_logging import logger
+
+
+class Index:
+    def __init__(self, _internal_state: IndexInfo):
+        self._state = _internal_state
+
+    def __repr__(self) -> str:
+        return f'Index: {self._state["resource_uri"]}'
+
+    @property
+    def uri(self) -> str:
+        return self._state['resource_uri']
+
+    @property
+    def id(self) -> str:
+        return f'{self._state["group_uuid"]}.{self._state["resource_uuid"]}'.replace('-', '_')
+
+    @staticmethod
+    def get_by_uri(uri: str) -> 'Index':
+        return Index(_internal_state=get_index_from_uri(uri))
+
+    @staticmethod
+    def create(uri: str, config: Dict[str, Any]) -> str:
+        BiolibApiClient.assert_is_signed_in(authenticated_action_description='create an Index')
+
+        response = api.client.post(
+            path='/resources/indexes/',
+            data={
+                'uri': uri,
+                'index_config': config,
+            },
+        )
+        result = response.json()
+        created_uri: str = result['uri']
+        logger.info(f"Successfully created Index '{created_uri}'")
+        return created_uri
+
+    @staticmethod
+    def create_from_config_file(uri: str, config_path: str) -> str:
+        with open(config_path) as config_file:
+            index_config = json.load(config_file)
+
+        return Index.create(uri=uri, config=index_config)

biolib/_index/types.py

@@ -0,0 +1,7 @@
+from typing import TypedDict
+
+
+class IndexInfo(TypedDict):
+    resource_uri: str
+    resource_uuid: str
+    group_uuid: str

biolib/_internal/data_record/data_record.py

@@ -1,7 +1,7 @@
 import sqlite3
 from pathlib import Path
 
-from biolib._internal.types.data_record import SqliteV1DatabaseSchema
+from biolib._shared.types import SqliteV1DatabaseSchema
 from biolib.api import client as api_client
 from biolib.biolib_api_client import AppGetResponse
 from biolib.biolib_api_client.biolib_app_api import _get_app_uri_from_str

biolib/_internal/data_record/push_data.py

@@ -1,10 +1,58 @@
+from __future__ import annotations
+
 import os
+from typing import Callable, Iterator
 
-from biolib import utils
+import biolib.api as api
 from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
-from biolib._internal.types.typing import List, Optional, Tuple
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
+from biolib.typing_utils import List, Optional, Tuple
+from biolib.utils import MultiPartUploader
+
+
+def _upload_from_iterator(
+    payload_iterator: Iterator[bytes],
+    payload_size_in_bytes: int,
+    resource_uuid: Optional[str] = None,
+    resource_version_uuid: Optional[str] = None,
+    use_process_pool: bool = False,
+    publish: bool = False,
+    on_progress: Optional[Callable[[int, int], None]] = None,
+) -> str:
+    if (resource_uuid is None) == (resource_version_uuid is None):
+        raise ValueError('Must provide exactly one of resource_uuid or resource_version_uuid')
+
+    if resource_version_uuid is None:
+        response = api.client.post(
+            path='/lfs/versions/',
+            data={'resource_uuid': resource_uuid},
+        )
+        resource_version_uuid = response.json()['uuid']
+
+    multipart_uploader = MultiPartUploader(
+        use_process_pool=use_process_pool,
+        get_presigned_upload_url_request={
+            'headers': None,
+            'requires_biolib_auth': True,
+            'path': f'/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
+        },
+        complete_upload_request={
+            'headers': None,
+            'requires_biolib_auth': True,
+            'path': f'/lfs/versions/{resource_version_uuid}/complete_upload/',
+        },
+        on_progress=on_progress,
+    )
+    multipart_uploader.upload(payload_iterator=payload_iterator, payload_size_in_bytes=payload_size_in_bytes)
+
+    if publish:
+        api.client.patch(
+            path=f'/resources/versions/{resource_version_uuid}/',
+            data={'state': 'published', 'set_as_active': True},
+        )
+
+    return resource_version_uuid
 
 
 def validate_data_path_and_get_files_and_size_of_directory(data_path: str) -> Tuple[List[str], int]:
@@ -28,9 +76,14 @@ def push_data_path(
     data_path: str,
     data_size_in_bytes: int,
     files_to_zip: List[str],
-    resource_version_uuid: str,
+    resource_uuid: Optional[str] = None,
+    resource_version_uuid: Optional[str] = None,
     chunk_size_in_mb: Optional[int] = None,
-) -> None:
+    publish: bool = False,
+) -> str:
+    if (resource_uuid is None) == (resource_version_uuid is None):
+        raise ValueError('Must provide exactly one of resource_uuid or resource_version_uuid')
+
     original_working_dir = os.getcwd()
     os.chdir(data_path)
 
@@ -49,19 +102,15 @@ def push_data_path(
     logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
 
     iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
-    multipart_uploader = utils.MultiPartUploader(
+
+    new_resource_version_uuid = _upload_from_iterator(
+        payload_iterator=iterable_zip_stream,
+        payload_size_in_bytes=data_size_in_bytes,
+        resource_uuid=resource_uuid,
+        resource_version_uuid=resource_version_uuid,
         use_process_pool=True,
-        get_presigned_upload_url_request=dict(
-            headers=None,
-            requires_biolib_auth=True,
-            path=f'/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
-        ),
-        complete_upload_request=dict(
-            headers=None,
-            requires_biolib_auth=True,
-            path=f'/lfs/versions/{resource_version_uuid}/complete_upload/',
-        ),
+        publish=publish,
     )
 
-    multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
     os.chdir(original_working_dir)
+    return new_resource_version_uuid

biolib/_internal/data_record/remote_storage_endpoint.py

@@ -1,5 +1,5 @@
 import os
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from urllib.parse import urlparse
 
 from biolib.api import client as api_client
@@ -16,7 +16,7 @@ class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
         self._presigned_url: Optional[str] = None
 
     def get_remote_url(self) -> str:
-        if not self._presigned_url or not self._expires_at or datetime.utcnow() > self._expires_at:
+        if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
             lfs_version: DataRecordVersion = api_client.get(
                 path=f'/lfs/versions/{self._resource_version_uuid}/',
             ).json()
@@ -29,7 +29,7 @@ class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
             else:
                 self._presigned_url = lfs_version['presigned_download_url']
 
-            self._expires_at = datetime.utcnow() + timedelta(minutes=8)
+            self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
             logger.debug(
                 f'DataRecord "{self._resource_version_uuid}" fetched presigned URL '
                 f'with expiry at {self._expires_at.isoformat()}'

biolib/_internal/file_utils.py

@@ -1,6 +1,7 @@
 import hashlib
 import io
 import os
+import posixpath
 import zipfile as zf
 from pathlib import Path
 
@@ -114,9 +115,11 @@ def path_to_renamed_path(path_str: str, prefix_with_slash: bool = True) -> str:
 
     if prefix_with_slash:
         if not result.startswith('/'):
-            return '/' + result
-        return result
+            result = '/' + result
+        # Normalize to handle cases like '/./mydir' -> '/mydir' and remove trailing slashes.
+        # Required because downstream Mappings class does exact string-prefix matching.
+        return posixpath.normpath(result)
     else:
         if result.startswith('/'):
-            return result[1:]
-        return result
+            result = result[1:]
+        return posixpath.normpath(result)

biolib/_internal/index/__init__.py

@@ -0,0 +1 @@
+from .index import get_index_from_uri

biolib/_internal/index/index.py

@@ -0,0 +1,18 @@
+from typing import Any, Dict
+
+from biolib._index.types import IndexInfo
+from biolib.api import client as api_client
+from biolib.biolib_api_client.biolib_app_api import _get_app_uri_from_str
+
+
+def get_index_from_uri(uri: str) -> IndexInfo:
+    normalized_uri = _get_app_uri_from_str(uri)
+    app_response: Dict[str, Any] = api_client.get(path='/app/', params={'uri': normalized_uri}).json()
+    resource_uri = app_response['app_version']['app_uri']
+    if app_response['app']['type'] != 'index':
+        raise Exception(f'Resource "{resource_uri}" is not an Index')
+    return IndexInfo(
+        resource_uri=app_response['app_version']['app_uri'],
+        resource_uuid=app_response['app']['public_id'],
+        group_uuid=app_response['app']['group_uuid'],
+    )

biolib/_internal/lfs/cache.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from biolib.biolib_logging import logger_no_user_data
 from biolib.compute_node.job_worker.cache_state import LfsCacheState
@@ -9,7 +9,7 @@ from biolib.compute_node.job_worker.cache_state import LfsCacheState
 def prune_lfs_cache(dry_run: bool) -> None:
     logger_no_user_data.info(f'Pruning LFS cache (dry run = {dry_run})...')
 
-    current_time = datetime.utcnow()
+    current_time = datetime.now(timezone.utc)
     paths_to_delete = set()
 
     with LfsCacheState() as state:
@@ -24,6 +24,8 @@ def prune_lfs_cache(dry_run: bool) -> None:
         lfs_uuids_to_keep_in_state = set()
         for lfs_uuid, lfs in state['large_file_systems'].items():
             last_used_at = datetime.fromisoformat(lfs['last_used_at'])
+            if last_used_at.tzinfo is None:
+                last_used_at = last_used_at.replace(tzinfo=timezone.utc)
             lfs_time_to_live_in_days = 60 if lfs['state'] == 'ready' else 7
 
             if last_used_at < current_time - timedelta(days=lfs_time_to_live_in_days):