pybiolib 1.1.2038__tar.gz → 1.1.2097__tar.gz

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.2038
+Version: 1.1.2097
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/__init__.py

@@ -13,7 +13,7 @@ from biolib.biolib_api_client import BiolibApiClient as _BioLibApiClient, App
 from biolib.jobs import Job as _Job
 from biolib import user as _user
 from biolib.typing_utils import List, Optional
-from biolib._internal.data_record import DataRecord as _DataRecord
+from biolib._data_record.data_record import DataRecord as _DataRecord
 
 import biolib.api
 import biolib.app
@@ -45,6 +45,10 @@ def get_job(job_id: str) -> _Job:
     return _Job.create_from_uuid(uuid=job_id)
 
 
+def get_data_record(uri: str) -> _DataRecord:
+    return _DataRecord.get_by_uri(uri)
+
+
 def fetch_jobs(count: int = 25) -> List[_Job]:
     return _Job.fetch_jobs(count)
 

{pybiolib-1.1.2038/biolib/_internal/data_record → pybiolib-1.1.2097/biolib/_data_record}/data_record.py

@@ -1,47 +1,50 @@
-import os
-from collections import namedtuple
-from datetime import datetime
-from fnmatch import fnmatch
-from struct import Struct
-from typing import Callable, Dict, List, Optional, Union, cast
-
-from biolib import lfs
+from biolib import api
+from biolib._internal.data_record import get_data_record_state_from_uri, push_data_record_version
 from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
 from biolib._internal.http_client import HttpClient
 from biolib.api import client as api_client
-from biolib.biolib_api_client import AppGetResponse
+from biolib.biolib_api_client import BiolibApiClient
+from biolib.biolib_api_client.lfs_types import DataRecordInfo, DataRecordVersionInfo
 from biolib.biolib_binary_format import LazyLoadedFile
 from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
 from biolib.biolib_logging import logger
+from biolib.typing_utils import Optional as _Optional
 from biolib.utils.app_uri import parse_app_uri
-from biolib.utils.zip.remote_zip import RemoteZip  # type: ignore
+from biolib.utils.zip.remote_zip import RemoteZip
+
+
+import os
+from collections import namedtuple
+from datetime import datetime
+from fnmatch import fnmatch
+from struct import Struct
+from typing import Callable, Dict, List, cast, Union
 
 PathFilter = Union[str, Callable[[str], bool]]
 
 
 class DataRecord:
-    def __init__(self, uri: str):
-        self._uri = uri
+    def __init__(self, _internal_state: DataRecordVersionInfo):
+        self._state = _internal_state
 
     def __repr__(self):
-        return f'DataRecord: {self._uri}'
+        return f'DataRecord: {self._state["resource_uri"]}'
 
     @property
     def uri(self) -> str:
-        return self._uri
+        return self._state['resource_uri']
 
     @property
     def name(self) -> str:
-        uri_parsed = parse_app_uri(self.uri, use_account_as_name_default=False)
+        uri_parsed = parse_app_uri(self._state['resource_uri'], use_account_as_name_default=False)
         if not uri_parsed['app_name']:
-            raise ValueError('Expected parameter "uri" to contain resource name')
+            raise ValueError('Expected parameter "resource_uri" to contain resource name')
 
         return uri_parsed['app_name']
 
-    def list_files(self, path_filter: Optional[PathFilter] = None) -> List[LazyLoadedFile]:
-        app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': self._uri}).json()
+    def list_files(self, path_filter: _Optional[PathFilter] = None) -> List[LazyLoadedFile]:
         remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
-            resource_version_uuid=app_response['app_version']['public_id'],
+            resource_version_uuid=self._state['resource_version_uuid'],
         )
         files: List[LazyLoadedFile] = []
         with RemoteZip(url=remote_storage_endpoint.get_remote_url()) as remote_zip:
@@ -52,13 +55,12 @@ class DataRecord:
         return self._get_filtered_files(files=files, path_filter=path_filter) if path_filter else files
 
     def download_zip(self, output_path: str):
-        app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': self._uri}).json()
         remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
-            resource_version_uuid=app_response['app_version']['public_id'],
+            resource_version_uuid=self._state['resource_version_uuid'],
         )
         HttpClient.request(url=remote_storage_endpoint.get_remote_url(), response_path=output_path)
 
-    def download_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
+    def download_files(self, output_dir: str, path_filter: _Optional[PathFilter] = None) -> None:
         filtered_files = self.list_files(path_filter=path_filter)
 
         if len(filtered_files) == 0:
@@ -72,23 +74,51 @@ class DataRecord:
                 for chunk in file.get_data_iterator():
                     file_handle.write(chunk)
 
-    def save_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
+    def save_files(self, output_dir: str, path_filter: _Optional[PathFilter] = None) -> None:
         self.download_files(output_dir=output_dir, path_filter=path_filter)
 
-    def update(self, data_path: str) -> None:
+    def update(self, data_path: str, chunk_size_in_mb: _Optional[int] = None) -> None:
         assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
-        self._uri = lfs.push_large_file_system(lfs_uri=self._uri, input_dir=data_path)
+        uri = push_data_record_version(
+            data_record_uuid=self._state['resource_uuid'], input_dir=data_path, chunk_size_in_mb=chunk_size_in_mb
+        )
+        self._state = get_data_record_state_from_uri(uri)
 
     @staticmethod
-    def create(destination: str, data_path: str, name: Optional[str] = None) -> 'DataRecord':
-        assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
-        record_name = name if name else 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
-        record_uri = lfs.create_large_file_system(lfs_uri=f'{destination}/{record_name}')
-        record_version_uri = lfs.push_large_file_system(lfs_uri=record_uri, input_dir=data_path)
-        return DataRecord(uri=record_version_uri)
+    def get_by_uri(uri: str) -> 'DataRecord':
+        return DataRecord(_internal_state=get_data_record_state_from_uri(uri))
 
     @staticmethod
-    def fetch(uri: Optional[str] = None, count: Optional[int] = None) -> List['DataRecord']:
+    def create(destination: str, data_path: _Optional[str] = None) -> 'DataRecord':
+        BiolibApiClient.assert_is_signed_in(authenticated_action_description='create a Data Record')
+        if data_path is not None:
+            assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
+        uri_parsed = parse_app_uri(destination, use_account_as_name_default=False)
+        if uri_parsed['app_name_normalized']:
+            data_record_uri = destination
+        else:
+            record_name = 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
+            data_record_uri = f'{destination}/{record_name}'
+
+        uri_parsed = parse_app_uri(data_record_uri)
+        response = api.client.post(
+            path='/lfs/',
+            data={
+                'account_handle': uri_parsed['account_handle_normalized'],
+                'name': uri_parsed['app_name'],
+            },
+        )
+        data_record: DataRecordInfo = response.json()
+        logger.info(f"Successfully created new Data Record '{data_record['uri']}'")
+
+        if data_path is not None:
+            record_version_uri = push_data_record_version(data_record_uuid=data_record['uuid'], input_dir=data_path)
+            return DataRecord.get_by_uri(uri=record_version_uri)
+        else:
+            return DataRecord.get_by_uri(uri=data_record_uri)
+
+    @staticmethod
+    def fetch(uri: _Optional[str] = None, count: _Optional[int] = None) -> List['DataRecord']:
         max_page_size = 1_000
         params: Dict[str, Union[str, int]] = {
             'page_size': str(count or max_page_size),
@@ -106,7 +136,16 @@ class DataRecord:
                 f'Fetch results exceeded maximum count of {max_page_size}. Some data records might not be fetched.'
             )
 
-        return [DataRecord(result['resource_uri']) for result in results]
+        return [
+            DataRecord(
+                _internal_state={
+                    'resource_uri': result['resource_uri'],
+                    'resource_uuid': result['public_id'],
+                    'resource_version_uuid': result['active_version'],
+                }
+            )
+            for result in results
+        ]
 
     @staticmethod
     def _get_file(remote_storage_endpoint: DataRecordRemoteStorageEndpoint, file_info: Dict) -> LazyLoadedFile:

pybiolib-1.1.2097/biolib/_internal/data_record/__init__.py

@@ -0,0 +1 @@
+from .data_record import get_data_record_state_from_uri, push_data_record_version

pybiolib-1.1.2097/biolib/_internal/data_record/data_record.py

@@ -0,0 +1,73 @@
+import os
+from typing import Optional
+
+from biolib import api, utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
+from biolib.api import client as api_client
+from biolib.biolib_api_client import AppGetResponse, BiolibApiClient
+from biolib.biolib_api_client.lfs_types import DataRecordVersion, DataRecordVersionInfo
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger
+
+
+def push_data_record_version(data_record_uuid: str, input_dir: str, chunk_size_in_mb: Optional[int] = None) -> str:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='push data to a Data Record')
+
+    if not os.path.isdir(input_dir):
+        raise BioLibError(f'Could not find folder at {input_dir}')
+
+    if os.path.realpath(input_dir) == '/':
+        raise BioLibError('Pushing your root directory is not possible')
+
+    original_working_dir = os.getcwd()
+    os.chdir(input_dir)
+    files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
+
+    if data_size_in_bytes > 4_500_000_000_000:
+        raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+
+    min_chunk_size_bytes = 10_000_000
+    chunk_size_in_bytes: int
+    if chunk_size_in_mb:
+        chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
+        if chunk_size_in_bytes < min_chunk_size_bytes:
+            logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
+            chunk_size_in_bytes = min_chunk_size_bytes
+    else:
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
+
+    data_size_in_mb = round(data_size_in_bytes / 10**6)
+    print(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
+
+    response = api.client.post(path='/lfs/versions/', data={'resource_uuid': data_record_uuid})
+    data_record_version: DataRecordVersion = response.json()
+    iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
+
+    multipart_uploader = utils.MultiPartUploader(
+        use_process_pool=True,
+        get_presigned_upload_url_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f"/lfs/versions/{data_record_version['uuid']}/presigned_upload_url/",
+        ),
+        complete_upload_request=dict(
+            headers=None,
+            requires_biolib_auth=True,
+            path=f"/lfs/versions/{data_record_version['uuid']}/complete_upload/",
+        ),
+    )
+
+    multipart_uploader.upload(payload_iterator=iterable_zip_stream, payload_size_in_bytes=data_size_in_bytes)
+    os.chdir(original_working_dir)
+    logger.info(f"Successfully pushed a new Data Record version '{data_record_version['uri']}'")
+    return data_record_version['uri']
+
+
+def get_data_record_state_from_uri(uri) -> 'DataRecordVersionInfo':
+    app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': uri}).json()
+    return DataRecordVersionInfo(
+        resource_uri=app_response['app_version']['app_uri'],
+        resource_uuid=app_response['app']['public_id'],
+        resource_version_uuid=app_response['app_version']['public_id'],
+    )

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/_internal/data_record/remote_storage_endpoint.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from urllib.parse import urlparse
 
 from biolib.api import client as api_client
-from biolib.biolib_api_client.lfs_types import LargeFileSystemVersion
+from biolib.biolib_api_client.lfs_types import DataRecordVersion
 from biolib.biolib_binary_format.utils import RemoteEndpoint
 from biolib.biolib_logging import logger
 from biolib.typing_utils import Optional
@@ -17,7 +17,7 @@ class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
 
     def get_remote_url(self) -> str:
         if not self._presigned_url or not self._expires_at or datetime.utcnow() > self._expires_at:
-            lfs_version: LargeFileSystemVersion = api_client.get(
+            lfs_version: DataRecordVersion = api_client.get(
                 path=f'/lfs/versions/{self._resource_version_uuid}/',
             ).json()
 

pybiolib-1.1.2097/biolib/_internal/file_utils.py

@@ -0,0 +1,77 @@
+import io
+import os
+import zipfile as zf
+from pathlib import Path
+
+from biolib.typing_utils import Iterator, List, Tuple
+
+
+def get_files_and_size_of_directory(directory: str) -> Tuple[List[str], int]:
+    data_size = 0
+    file_list: List[str] = []
+
+    for path, _, files in os.walk(directory):
+        for file in files:
+            file_path = os.path.join(path, file)
+            if os.path.islink(file_path):
+                continue  # skip symlinks
+
+            relative_file_path = file_path[len(directory) + 1 :]  # +1 to remove starting slash
+            file_list.append(relative_file_path)
+            data_size += os.path.getsize(file_path)
+
+    return file_list, data_size
+
+
+def get_iterable_zip_stream(files: List[str], chunk_size: int) -> Iterator[bytes]:
+    class ChunkedIOBuffer(io.RawIOBase):
+        def __init__(self, chunk_size: int):
+            super().__init__()
+            self.chunk_size = chunk_size
+            self.tmp_data = bytearray()
+
+        def get_buffer_size(self):
+            return len(self.tmp_data)
+
+        def read_chunk(self):
+            chunk = bytes(self.tmp_data[: self.chunk_size])
+            self.tmp_data = self.tmp_data[self.chunk_size :]
+            return chunk
+
+        def write(self, data):
+            data_length = len(data)
+            self.tmp_data += data
+            return data_length
+
+    # create chunked buffer to hold data temporarily
+    io_buffer = ChunkedIOBuffer(chunk_size)
+
+    # create zip writer that will write to the io buffer
+    zip_writer = zf.ZipFile(io_buffer, mode='w')  # type: ignore
+
+    for file_path in files:
+        # generate zip info and prepare zip pointer for writing
+        z_info = zf.ZipInfo.from_file(file_path)
+        zip_pointer = zip_writer.open(z_info, mode='w')
+        if Path(file_path).is_file():
+            # read file chunk by chunk
+            with open(file_path, 'br') as file_pointer:
+                while True:
+                    chunk = file_pointer.read(chunk_size)
+                    if len(chunk) == 0:
+                        break
+                    # write the chunk to the zip
+                    zip_pointer.write(chunk)
+                    # if writing the chunk caused us to go over chunk_size, flush it
+                    if io_buffer.get_buffer_size() > chunk_size:
+                        yield io_buffer.read_chunk()
+
+        zip_pointer.close()
+
+    # flush any remaining data in the stream (e.g. zip file meta data)
+    zip_writer.close()
+    while True:
+        chunk = io_buffer.read_chunk()
+        if len(chunk) == 0:
+            break
+        yield chunk

pybiolib-1.1.2097/biolib/_internal/lfs/__init__.py

@@ -0,0 +1 @@
+from .cache import prune_lfs_cache

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/_internal/push_application.py

@@ -6,12 +6,12 @@ import rich.progress
 import yaml
 
 from biolib import api, utils
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
 from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger
-from biolib.lfs.utils import get_files_and_size_of_directory, get_iterable_zip_stream
 from biolib.typing_utils import Iterable, Optional, Set, TypedDict
 
 REGEX_MARKDOWN_INLINE_IMAGE = re.compile(r'!\[(?P<alt>.*)\]\((?P<src>.*)\)')

pybiolib-1.1.2097/biolib/_internal/runtime.py

@@ -0,0 +1,19 @@
+from biolib.typing_utils import TypedDict
+
+
+class RuntimeJobDataDict(TypedDict):
+    version: str
+    job_requested_machine: str
+    job_uuid: str
+    job_auth_token: str
+    app_uri: str
+
+
+class BioLibRuntimeError(Exception):
+    pass
+
+
+class BioLibRuntimeNotRecognizedError(BioLibRuntimeError):
+    def __init__(self, message='The runtime is not recognized as a BioLib app'):
+        self.message = message
+        super().__init__(self.message)

{pybiolib-1.1.2038/biolib/_internal → pybiolib-1.1.2097/biolib/_runtime}/runtime.py

@@ -1,30 +1,13 @@
-import json
-import re
-
 from biolib import api
-from biolib.typing_utils import Optional, TypedDict, cast
-
-
-class RuntimeJobDataDict(TypedDict):
-    version: str
-    job_requested_machine: str
-    job_uuid: str
-    job_auth_token: str
-    app_uri: str
+from biolib._internal.runtime import BioLibRuntimeError, BioLibRuntimeNotRecognizedError, RuntimeJobDataDict
+from biolib.typing_utils import cast, Optional as _Optional
 
-
-class BioLibRuntimeError(Exception):
-    pass
-
-
-class BioLibRuntimeNotRecognizedError(BioLibRuntimeError):
-    def __init__(self, message='The runtime is not recognized as a BioLib app'):
-        self.message = message
-        super().__init__(self.message)
+import json
+import re
 
 
 class Runtime:
-    _job_data: Optional[RuntimeJobDataDict] = None
+    _job_data: _Optional[RuntimeJobDataDict] = None
 
     @staticmethod
     def check_is_environment_biolib_app() -> bool:
@@ -73,7 +56,7 @@ class Runtime:
         api.client.post(data={'note': note}, path=f'/jobs/{job_id}/notes/')
 
     @staticmethod
-    def _try_to_get_job_data() -> Optional[RuntimeJobDataDict]:
+    def _try_to_get_job_data() -> _Optional[RuntimeJobDataDict]:
         if not Runtime._job_data:
             try:
                 with open('/biolib/secrets/biolib_system_secret') as file:

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/biolib_api_client/api_client.py

@@ -6,7 +6,7 @@ import os
 from datetime import datetime, timezone
 from json.decoder import JSONDecodeError
 
-from biolib._internal.runtime import Runtime
+from biolib._runtime.runtime import Runtime
 from biolib._internal.http_client import HttpClient
 from biolib.typing_utils import Optional
 from biolib.biolib_errors import BioLibError

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/biolib_api_client/app_types.py

@@ -16,6 +16,7 @@ class AppVersion(AppVersionSlim):
     source_code_license: str
     stdout_render_type: Literal['text', 'markdown']
     main_output_file: Optional[str]
+    app_uri: str
 
 
 class App(TypedDict):

pybiolib-1.1.2097/biolib/biolib_api_client/lfs_types.py

@@ -0,0 +1,19 @@
+from biolib.typing_utils import TypedDict
+
+
+class DataRecordVersion(TypedDict):
+    presigned_download_url: str
+    size_bytes: int
+    uri: str
+    uuid: str
+
+
+class DataRecordInfo(TypedDict):
+    uri: str
+    uuid: str
+
+
+class DataRecordVersionInfo(TypedDict):
+    resource_uri: str
+    resource_uuid: str
+    resource_version_uuid: str

pybiolib-1.1.2097/biolib/cli/data_record.py

@@ -0,0 +1,79 @@
+import json
+import logging
+import os
+from typing import Dict, List
+
+import click
+
+from biolib._data_record.data_record import DataRecord
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.typing_utils import Optional
+
+
+@click.group(help='Data Records')
+def data_record() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@data_record.command(help='Create a Data Record')
+@click.argument('uri', required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+def create(uri: str, data_path: str) -> None:
+    DataRecord.create(destination=uri, data_path=data_path)
+
+
+@data_record.command(help='Update a Data Record')
+@click.argument('uri', required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+@click.option('--chunk-size', default=None, required=False, type=click.INT, help='The size of each chunk (In MB)')
+def update(uri: str, data_path: str, chunk_size: Optional[int]) -> None:
+    DataRecord.get_by_uri(uri=uri).update(data_path=data_path, chunk_size_in_mb=chunk_size)
+
+
+@data_record.command(help='Download files from a Data Record')
+@click.argument('uri', required=True)
+@click.option('--file', required=False, type=str)
+@click.option('--path-filter', required=False, type=str, hide_input=True)
+def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+    if file is not None:
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
+    else:
+        assert not os.path.exists(record.name), f'Directory with name {record.name} already exists in current directory'
+        record.save_files(output_dir=record.name, path_filter=path_filter)
+
+
+@data_record.command(help='Describe a Data Record')
+@click.argument('uri', required=True)
+@click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
+def describe(uri: str, output_as_json: bool) -> None:
+    record = DataRecord.get_by_uri(uri)
+    files_info: List[Dict] = []
+    total_size_in_bytes = 0
+    for file in record.list_files():
+        files_info.append({'path': file.path, 'size_bytes': file.length})
+        total_size_in_bytes += file.length
+
+    if output_as_json:
+        print(
+            json.dumps(
+                obj={'uri': record.uri, 'size_bytes': total_size_in_bytes, 'files': files_info},
+                indent=4,
+            )
+        )
+    else:
+        print(f'Data Record {record.uri}\ntotal {total_size_in_bytes} bytes\n')
+        print('size bytes    path')
+        for file_info in files_info:
+            size_string = str(file_info['size_bytes'])
+            leading_space_string = ' ' * (10 - len(size_string))
+            print(f"{leading_space_string}{size_string}    {file_info['path']}")

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/cli/lfs.py

@@ -7,9 +7,9 @@ from typing import Dict, List
 import click
 
 from biolib import biolib_errors
-from biolib._internal.data_record import DataRecord
+from biolib._data_record.data_record import DataRecord
+from biolib._internal.lfs import prune_lfs_cache
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.lfs import create_large_file_system, prune_lfs_cache, push_large_file_system
 from biolib.typing_utils import Optional
 
 
@@ -21,9 +21,10 @@ def lfs() -> None:
 @lfs.command(help='Create a Large File System')
 @click.argument('uri', required=True)
 def create(uri: str) -> None:
+    logger.warning('This is command deprecated, please use "biolib data-record create" instead.')
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
-    create_large_file_system(lfs_uri=uri)
+    DataRecord.create(destination=uri)
 
 
 @lfs.command(help='Push a new version of a Large File System')
@@ -31,10 +32,11 @@ def create(uri: str) -> None:
 @click.option('--path', required=True, type=click.Path(exists=True))
 @click.option('--chunk-size', default=None, required=False, type=click.INT, help='The size of each chunk (In MB)')
 def push(uri: str, path: str, chunk_size: Optional[int]) -> None:
+    logger.warning('This is command deprecated, please use "biolib data-record update" instead.')
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     try:
-        push_large_file_system(lfs_uri=uri, input_dir=path, chunk_size_in_mb=chunk_size)
+        DataRecord.get_by_uri(uri=uri).update(data_path=path, chunk_size_in_mb=chunk_size)
     except biolib_errors.BioLibError as error:
         print(f'An error occurred:\n{error.message}', file=sys.stderr)
         exit(1)
@@ -44,10 +46,11 @@ def push(uri: str, path: str, chunk_size: Optional[int]) -> None:
 @click.argument('uri', required=True)
 @click.option('--file-path', required=True, type=str)
 def download_file(uri: str, file_path: str) -> None:
+    logger.warning('This is command deprecated, please use "biolib data-record download" instead.')
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     try:
-        record = DataRecord(uri=uri)
+        record = DataRecord.get_by_uri(uri=uri)
         try:
             file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file_path][0]
         except IndexError:
@@ -66,7 +69,8 @@ def download_file(uri: str, file_path: str) -> None:
 @click.argument('uri', required=True)
 @click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
 def describe(uri: str, output_as_json: bool) -> None:
-    data_record = DataRecord(uri)
+    logger.warning('This is command deprecated, please use "biolib data-record describe" instead.')
+    data_record = DataRecord.get_by_uri(uri)
     files_info: List[Dict] = []
     total_size_in_bytes = 0
     for file in data_record.list_files():

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/runtime/__init__.py

@@ -1,5 +1,5 @@
 import warnings
-from biolib.sdk import Runtime as _Runtime
+from biolib._runtime.runtime import Runtime as _Runtime
 
 
 def set_main_result_prefix(result_prefix: str) -> None:

{pybiolib-1.1.2038 → pybiolib-1.1.2097}/biolib/sdk/__init__.py

@@ -1,12 +1,12 @@
 # Imports to hide and use as private internal utils
+from biolib._data_record.data_record import DataRecord as _DataRecord
 from biolib._internal.push_application import push_application as _push_application
 from biolib._internal.push_application import set_app_version_as_active as _set_app_version_as_active
 from biolib.app import BioLibApp as _BioLibApp
 from biolib.typing_utils import Optional as _Optional
 
-# Imports to expose as public API
-from biolib._internal.data_record import DataRecord
-from biolib._internal.runtime import Runtime
+# Classes to expose as public API
+from biolib._runtime.runtime import Runtime
 
 
 def push_app_version(uri: str, path: str) -> _BioLibApp:
@@ -42,5 +42,9 @@ def get_app_version_pytest_plugin(app_version: _BioLibApp):
     return AppVersionFixturePlugin(app_version)
 
 
-def create_data_record(destination: str, data_path: str, name: _Optional[str] = None) -> DataRecord:
-    return DataRecord.create(destination, data_path, name)
+def create_data_record(destination: str, data_path: str, name: _Optional[str] = None) -> _DataRecord:
+    if name:
+        destination_with_name = f"{destination}/{name}"
+    else:
+        destination_with_name = destination
+    return _DataRecord.create(destination_with_name, data_path)