pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/_internal/data_record/data_record.py

@@ -0,0 +1,85 @@
+import sqlite3
+from pathlib import Path
+
+from biolib._shared.types import SqliteV1DatabaseSchema
+
+
+def get_actual_schema(db_path):
+    if not db_path.exists():
+        raise Exception(f'File {db_path} not found.')
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+
+    actual_schema: SqliteV1DatabaseSchema = {'tables': {}}
+    for table in tables:
+        table_name = table[0]
+        cursor.execute(f'PRAGMA table_info({table_name});')
+        columns = cursor.fetchall()
+        actual_schema['tables'][table_name] = {'columns': {}}
+        for column in columns:
+            actual_schema['tables'][table_name]['columns'][column[1]] = {
+                'type': column[2],
+                'nullable': not bool(column[3]),
+            }
+
+        cursor.execute(f'PRAGMA foreign_key_list({table_name});')
+        foreign_keys = cursor.fetchall()
+        for fk in foreign_keys:
+            actual_schema['tables'][table_name]['columns'][fk[3]]['foreign_key'] = {'table': fk[2], 'column': fk[4]}
+
+    conn.close()
+    return actual_schema
+
+
+def verify_schema(specification: SqliteV1DatabaseSchema, actual_schema: SqliteV1DatabaseSchema):
+    for table_name, table_spec in specification['tables'].items():
+        if table_name not in actual_schema['tables']:
+            raise Exception(f"Error: Table '{table_name}' is missing.")
+
+        for column_name, column_spec in table_spec['columns'].items():
+            if column_name not in actual_schema['tables'][table_name]['columns']:
+                raise Exception(f"Error: Column '{column_name}' in table '{table_name}' is missing.")
+
+            actual_column = actual_schema['tables'][table_name]['columns'][column_name]
+            if actual_column['type'] != column_spec['type']:
+                raise Exception(
+                    f"Error: Column '{column_name}' in table '{table_name}' "
+                    "has type '{actual_column['type']}' but expected '{column_spec['type']}'."
+                )
+
+            if not actual_column['nullable'] and column_spec.get('nullable', True):
+                raise Exception(
+                    f"Error: Column '{column_name}' in table '{table_name}' is "
+                    'not nullable but should be nullable according to the specification.'
+                )
+
+        for column_name, column_spec in table_spec['columns'].items():
+            if column_spec.get('foreign_key'):
+                foreign_key_spec = column_spec['foreign_key']
+                if actual_schema['tables'][table_name]['columns'][column_name].get('foreign_key'):
+                    fk = actual_schema['tables'][table_name]['columns'][column_name]['foreign_key']
+                    if (
+                        fk
+                        and foreign_key_spec
+                        and fk['table'] == foreign_key_spec['table']
+                        and fk['column'] == foreign_key_spec['column']
+                    ):
+                        raise Exception(
+                            f"Error: Column '{column_name}' in table '{table_name}' does "
+                            'not have the correct foreign key constraint.'
+                        )
+                else:
+                    raise Exception(
+                        f"Error: Column '{column_name}' in table '{table_name}' does "
+                        'not have a foreign key constraint.'
+                    )
+
+
+def validate_sqlite_v1(schema: SqliteV1DatabaseSchema, sqlite_file: Path):
+    actual_schema = get_actual_schema(sqlite_file)
+    print(schema)
+    print(actual_schema)
+    verify_schema(specification=schema, actual_schema=actual_schema)

biolib/_internal/data_record/push_data.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import os
+from typing import Callable, Iterator
+
+import biolib.api as api
+from biolib._internal.file_utils import get_files_and_size_of_directory, get_iterable_zip_stream
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger
+from biolib.typing_utils import List, Optional, Tuple
+from biolib.utils import MultiPartUploader
+
+
+def _upload_from_iterator(
+    payload_iterator: Iterator[bytes],
+    payload_size_in_bytes: int,
+    resource_uuid: Optional[str] = None,
+    resource_version_uuid: Optional[str] = None,
+    use_process_pool: bool = False,
+    publish: bool = False,
+    on_progress: Optional[Callable[[int, int], None]] = None,
+) -> str:
+    if (resource_uuid is None) == (resource_version_uuid is None):
+        raise ValueError('Must provide exactly one of resource_uuid or resource_version_uuid')
+
+    if resource_version_uuid is None:
+        response = api.client.post(
+            path='/lfs/versions/',
+            data={'resource_uuid': resource_uuid},
+        )
+        resource_version_uuid = response.json()['uuid']
+
+    multipart_uploader = MultiPartUploader(
+        use_process_pool=use_process_pool,
+        get_presigned_upload_url_request={
+            'headers': None,
+            'requires_biolib_auth': True,
+            'path': f'/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
+        },
+        complete_upload_request={
+            'headers': None,
+            'requires_biolib_auth': True,
+            'path': f'/lfs/versions/{resource_version_uuid}/complete_upload/',
+        },
+        on_progress=on_progress,
+    )
+    multipart_uploader.upload(payload_iterator=payload_iterator, payload_size_in_bytes=payload_size_in_bytes)
+
+    if publish:
+        api.client.patch(
+            path=f'/resources/versions/{resource_version_uuid}/',
+            data={'state': 'published', 'set_as_active': True},
+        )
+
+    return resource_version_uuid
+
+
+def validate_data_path_and_get_files_and_size_of_directory(data_path: str) -> Tuple[List[str], int]:
+    assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
+
+    if os.path.realpath(data_path) == '/':
+        raise BioLibError('Pushing your root directory is not possible')
+
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+    files_to_zip, data_size_in_bytes = get_files_and_size_of_directory(directory=os.getcwd())
+    os.chdir(original_working_dir)
+
+    if data_size_in_bytes > 4_500_000_000_000:
+        raise BioLibError('Attempted to push directory with a size larger than the limit of 4.5 TB')
+
+    return files_to_zip, data_size_in_bytes
+
+
+def push_data_path(
+    data_path: str,
+    data_size_in_bytes: int,
+    files_to_zip: List[str],
+    resource_uuid: Optional[str] = None,
+    resource_version_uuid: Optional[str] = None,
+    chunk_size_in_mb: Optional[int] = None,
+    publish: bool = False,
+) -> str:
+    if (resource_uuid is None) == (resource_version_uuid is None):
+        raise ValueError('Must provide exactly one of resource_uuid or resource_version_uuid')
+
+    original_working_dir = os.getcwd()
+    os.chdir(data_path)
+
+    min_chunk_size_bytes = 10_000_000
+    chunk_size_in_bytes: int
+    if chunk_size_in_mb:
+        chunk_size_in_bytes = chunk_size_in_mb * 1_000_000  # Convert megabytes to bytes
+        if chunk_size_in_bytes < min_chunk_size_bytes:
+            logger.warning('Specified chunk size is too small, using minimum of 10 MB instead.')
+            chunk_size_in_bytes = min_chunk_size_bytes
+    else:
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        chunk_size_in_bytes = max(min_chunk_size_bytes, int(data_size_in_bytes / 9_000))
+
+    data_size_in_mb = round(data_size_in_bytes / 10**6)
+    logger.info(f'Zipping {len(files_to_zip)} files, in total ~{data_size_in_mb}mb of data')
+
+    iterable_zip_stream = get_iterable_zip_stream(files=files_to_zip, chunk_size=chunk_size_in_bytes)
+
+    new_resource_version_uuid = _upload_from_iterator(
+        payload_iterator=iterable_zip_stream,
+        payload_size_in_bytes=data_size_in_bytes,
+        resource_uuid=resource_uuid,
+        resource_version_uuid=resource_version_uuid,
+        use_process_pool=True,
+        publish=publish,
+    )
+
+    os.chdir(original_working_dir)
+    return new_resource_version_uuid

biolib/_internal/data_record/remote_storage_endpoint.py

@@ -0,0 +1,43 @@
+import os
+from datetime import datetime, timedelta, timezone
+from urllib.parse import urlparse
+
+from biolib._shared.types import ResourceDetailedDict
+from biolib.api import client as api_client
+from biolib.biolib_binary_format.utils import RemoteEndpoint
+from biolib.biolib_logging import logger
+from biolib.typing_utils import Optional
+
+
+class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
+    def __init__(self, uri: str):
+        self._uri: str = uri
+        self._expires_at: Optional[datetime] = None
+        self._presigned_url: Optional[str] = None
+
+    def get_remote_url(self) -> str:
+        if not self._presigned_url or not self._expires_at or datetime.now(timezone.utc) > self._expires_at:
+            resource_response: ResourceDetailedDict = api_client.get(
+                path='/resource/',
+                params={'uri': self._uri},
+            ).json()
+
+            version = resource_response.get('version')
+            assets = version.get('assets') if version else None
+            if not assets:
+                raise Exception(f'Resource "{self._uri}" has no downloadable assets')
+
+            download_url = assets['download_url']
+            app_caller_proxy_job_storage_base_url = os.getenv('BIOLIB_CLOUD_JOB_STORAGE_BASE_URL', '')
+            if app_caller_proxy_job_storage_base_url:
+                parsed_url = urlparse(download_url)
+                self._presigned_url = f'{app_caller_proxy_job_storage_base_url}{parsed_url.path}?{parsed_url.query}'
+            else:
+                self._presigned_url = download_url
+
+            self._expires_at = datetime.now(timezone.utc) + timedelta(minutes=8)
+            logger.debug(
+                f'DataRecord "{self._uri}" fetched presigned URL ' f'with expiry at {self._expires_at.isoformat()}'
+            )
+
+        return self._presigned_url

biolib/_internal/errors.py

@@ -0,0 +1,5 @@
+from biolib.biolib_errors import BioLibError
+
+
+class AuthenticationError(BioLibError):
+    """Raised when authentication is required but user is not signed in."""

biolib/_internal/file_utils.py

@@ -0,0 +1,125 @@
+import hashlib
+import io
+import os
+import posixpath
+import zipfile as zf
+from pathlib import Path
+
+from biolib.typing_utils import Iterator, List, Tuple
+
+
+def get_files_and_size_of_directory(directory: str) -> Tuple[List[str], int]:
+    data_size = 0
+    file_list: List[str] = []
+
+    for path, _, files in os.walk(directory):
+        for file in files:
+            file_path = os.path.join(path, file)
+            if os.path.islink(file_path):
+                continue  # skip symlinks
+
+            relative_file_path = file_path[len(directory) + 1 :]  # +1 to remove starting slash
+            file_list.append(relative_file_path)
+            data_size += os.path.getsize(file_path)
+
+    return file_list, data_size
+
+
+def get_iterable_zip_stream(files: List[str], chunk_size: int) -> Iterator[bytes]:
+    class ChunkedIOBuffer(io.RawIOBase):
+        def __init__(self, chunk_size: int):
+            super().__init__()
+            self.chunk_size = chunk_size
+            self.tmp_data = bytearray()
+
+        def get_buffer_size(self):
+            return len(self.tmp_data)
+
+        def read_chunk(self):
+            chunk = bytes(self.tmp_data[: self.chunk_size])
+            self.tmp_data = self.tmp_data[self.chunk_size :]
+            return chunk
+
+        def write(self, data):
+            data_length = len(data)
+            self.tmp_data += data
+            return data_length
+
+    # create chunked buffer to hold data temporarily
+    io_buffer = ChunkedIOBuffer(chunk_size)
+
+    # create zip writer that will write to the io buffer
+    zip_writer = zf.ZipFile(io_buffer, mode='w')  # type: ignore
+
+    for file_path in files:
+        # generate zip info and prepare zip pointer for writing
+        z_info = zf.ZipInfo.from_file(file_path)
+        zip_pointer = zip_writer.open(z_info, mode='w')
+        if Path(file_path).is_file():
+            # read file chunk by chunk
+            with open(file_path, 'br') as file_pointer:
+                while True:
+                    chunk = file_pointer.read(chunk_size)
+                    if len(chunk) == 0:
+                        break
+                    # write the chunk to the zip
+                    zip_pointer.write(chunk)
+                    # if writing the chunk caused us to go over chunk_size, flush it
+                    if io_buffer.get_buffer_size() > chunk_size:
+                        yield io_buffer.read_chunk()
+
+        zip_pointer.close()
+
+    # flush any remaining data in the stream (e.g. zip file meta data)
+    zip_writer.close()
+    while True:
+        chunk = io_buffer.read_chunk()
+        if len(chunk) == 0:
+            break
+        yield chunk
+
+
+def path_to_renamed_path(path_str: str, prefix_with_slash: bool = True) -> str:
+    """
+    Normalize file paths consistently:
+    - If path contains '..' (relative path going up), convert to absolute path
+    - If relative path not containing '..', keep as is, but prepend / if prefix_with_slash=True
+    - If absolute path that is subpath of current directory, convert to relative path
+    - If absolute path not subpath of current directory, hash the folder path and keep filename
+    """
+    path = Path(path_str)
+    current_dir = Path.cwd()
+
+    if '..' in path.parts:
+        resolved_path = path.resolve()
+        try:
+            relative_path = resolved_path.relative_to(current_dir)
+            result = str(relative_path)
+        except ValueError:
+            folder_path = str(resolved_path.parent)
+            filename = resolved_path.name
+            folder_hash = hashlib.md5(folder_path.encode()).hexdigest()[:6]
+            result = f'/{folder_hash}/{filename}'
+    elif path.is_absolute():
+        try:
+            resolved_path = path.resolve()
+            relative_path = resolved_path.relative_to(current_dir)
+            result = str(relative_path)
+        except ValueError:
+            folder_path = str(path.parent)
+            filename = path.name
+            folder_hash = hashlib.md5(folder_path.encode()).hexdigest()[:6]
+            result = f'/{folder_hash}/{filename}'
+    else:
+        result = path_str
+
+    if prefix_with_slash:
+        if not result.startswith('/'):
+            result = '/' + result
+        # Normalize to handle cases like '/./mydir' -> '/mydir' and remove trailing slashes.
+        # Required because downstream Mappings class does exact string-prefix matching.
+        return posixpath.normpath(result)
+    else:
+        if result.startswith('/'):
+            result = result[1:]
+        return posixpath.normpath(result)

biolib/_internal/fuse_mount/__init__.py

@@ -0,0 +1 @@
+from .experiment_fuse_mount import ExperimentFuseMount

biolib/_internal/fuse_mount/experiment_fuse_mount.py

@@ -0,0 +1,209 @@
+import errno
+import os
+import stat
+from datetime import datetime, timezone
+from time import time
+
+from biolib._internal.libs.fusepy import FUSE, FuseOSError, Operations
+from biolib.biolib_errors import BioLibError
+from biolib.jobs import Job
+from biolib.typing_utils import Dict, List, Optional, Tuple, TypedDict
+
+
+class _AttributeDict(TypedDict):
+    st_atime: int
+    st_ctime: int
+    st_gid: int
+    st_mode: int
+    st_mtime: int
+    st_nlink: int
+    st_size: int
+    st_uid: int
+
+
+_SUCCESS_CODE = 0
+
+
+class ExperimentFuseMount(Operations):
+    def __init__(self, experiment):
+        self._experiment = experiment
+        self._job_names_map: Optional[Dict[str, Job]] = None
+        self._jobs_last_fetched_at: float = 0.0
+        self._mounted_at_epoch_seconds: int = int(time())
+
+    @staticmethod
+    def mount_experiment(experiment, mount_path: str) -> None:
+        FUSE(
+            operations=ExperimentFuseMount(experiment),
+            mountpoint=mount_path,
+            nothreads=True,
+            foreground=True,
+            allow_other=False,
+        )
+
+    def getattr(self, path: str, fh=None) -> _AttributeDict:
+        if path == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
+
+        job, path_in_job = self._parse_path(path)
+        job_finished_at_epoch_seconds: int = int(
+            datetime.fromisoformat(job.to_dict()['finished_at'].rstrip('Z')).replace(tzinfo=timezone.utc).timestamp()
+        )
+
+        if path_in_job == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        try:
+            file = job.get_output_file(path_in_job)
+            return self._get_file_attributes(
+                timestamp_epoch_seconds=job_finished_at_epoch_seconds,
+                size_in_bytes=file.length,
+            )
+        except BioLibError:
+            # file not found
+            pass
+
+        file_paths_in_job = [file.path for file in job.list_output_files()]
+
+        for file_path_in_job in file_paths_in_job:
+            if file_path_in_job.startswith(path_in_job):
+                return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+    def readdir(self, path: str, fh: int) -> List[str]:
+        directory_entries = ['.', '..']
+
+        if path == '/':
+            directory_entries.extend(self._get_job_names_map(refresh_jobs=True).keys())
+        else:
+            job, path_in_job = self._parse_path(path)
+            dir_path_in_job = '/' if path_in_job == '/' else path_in_job + '/'
+            depth = dir_path_in_job.count('/')
+            directory_entries.extend(
+                set(
+                    [
+                        file.path.split('/')[depth]
+                        for file in job.list_output_files()
+                        if file.path.startswith(dir_path_in_job)
+                    ]
+                )
+            )
+
+        return directory_entries
+
+    def open(self, path: str, flags: int) -> int:
+        job, path_in_job = self._parse_path(path)
+        try:
+            job.get_output_file(path_in_job)
+        except BioLibError:
+            # file not found
+            raise FuseOSError(errno.ENOENT) from None
+
+        return 1234  # dummy file handle
+
+    def read(self, path: str, size: int, offset: int, fh: int) -> bytes:
+        job, path_in_job = self._parse_path(path)
+        try:
+            file = job.get_output_file(path_in_job)
+        except BioLibError:
+            raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+        return file.get_data(start=offset, length=size)
+
+    def release(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def releasedir(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def flush(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    @staticmethod
+    def _get_directory_attributes(timestamp_epoch_seconds: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFDIR | 0o555,  # Directory that is readable and executable by owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=1,
+            st_uid=os.getuid(),
+        )
+
+    @staticmethod
+    def _get_file_attributes(timestamp_epoch_seconds: int, size_in_bytes: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFREG | 0o444,  # Regular file with read permissions for owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=size_in_bytes,
+            st_uid=os.getuid(),
+        )
+
+    def _get_job_names_map(self, refresh_jobs=False) -> Dict[str, Job]:
+        current_time = time()
+        if not self._job_names_map or (current_time - self._jobs_last_fetched_at > 1 and refresh_jobs):
+            self._jobs_last_fetched_at = current_time
+            self._job_names_map = {job.get_name(): job for job in self._experiment.get_jobs(status='completed')}
+
+        return self._job_names_map
+
+    def _parse_path(self, path: str) -> Tuple[Job, str]:
+        path_splitted = path.split('/')
+        job_name = path_splitted[1]
+        path_in_job = '/' + '/'.join(path_splitted[2:])
+        job = self._get_job_names_map().get(job_name)
+        if not job:
+            raise FuseOSError(errno.ENOENT)  # No such file or directory
+
+        return job, path_in_job
+
+    # ----------------------------------- File system methods not implemented below -----------------------------------
+
+    def chmod(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def chown(self, path, uid, gid):
+        raise FuseOSError(errno.EACCES)
+
+    def mknod(self, path, mode, dev):
+        raise FuseOSError(errno.EACCES)
+
+    def rmdir(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def mkdir(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def unlink(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def symlink(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def rename(self, old, new):
+        raise FuseOSError(errno.EACCES)
+
+    def link(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def utimens(self, path, times=None):
+        raise FuseOSError(errno.EACCES)
+
+    def create(self, path, mode, fi=None):
+        raise FuseOSError(errno.EACCES)
+
+    def write(self, path, data, offset, fh):
+        raise FuseOSError(errno.EACCES)
+
+    def truncate(self, path, length, fh=None):
+        raise FuseOSError(errno.EACCES)
+
+    def fsync(self, path, datasync, fh):
+        raise FuseOSError(errno.EACCES)