pybiolib 1.1.1730__tar.gz → 1.1.1990__tar.gz

{pybiolib-1.1.1730 → pybiolib-1.1.1990}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.1730
+Version: 1.1.1990
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT
@@ -25,7 +25,6 @@ Requires-Dist: flask (>=2.0.1) ; extra == "compute-node"
 Requires-Dist: gunicorn (>=20.1.0) ; extra == "compute-node"
 Requires-Dist: importlib-metadata (>=1.6.1)
 Requires-Dist: pyyaml (>=5.3.1)
-Requires-Dist: requests (>=2.25.1)
 Requires-Dist: rich (>=12.4.4)
 Requires-Dist: typing_extensions (>=3.10.0) ; python_version < "3.8"
 Description-Content-Type: text/markdown

{pybiolib-1.1.1730 → pybiolib-1.1.1990}/biolib/__init__.py

@@ -13,10 +13,12 @@ from biolib.biolib_api_client import BiolibApiClient as _BioLibApiClient, App
 from biolib.jobs import Job as _Job
 from biolib import user as _user
 from biolib.typing_utils import List, Optional
+from biolib._internal.data_record import DataRecord as _DataRecord
 
 import biolib.api
 import biolib.app
 import biolib.cli
+import biolib.sdk
 import biolib.utils
 
 
@@ -33,8 +35,8 @@ def load(uri: str) -> _BioLibApp:
 def search(
         search_query: Optional[str] = None,
         team: Optional[str] = None,
-        count: int = 100
-    ) -> List[str]:
+        count: int = 100,
+) -> List[str]:
     apps: List[str] = search_apps(search_query, team, count)
     return apps
 
@@ -47,6 +49,10 @@ def fetch_jobs(count: int = 25) -> List[_Job]:
     return _Job.fetch_jobs(count)
 
 
+def fetch_data_records(uri: Optional[str] = None, count: Optional[int] = None) -> List[_DataRecord]:
+    return _DataRecord.fetch(uri, count)
+
+
 def get_experiment(name: str) -> Experiment:
     return Experiment(name)
 

pybiolib-1.1.1990/biolib/_internal/data_record/__init__.py

@@ -0,0 +1 @@
+from .data_record import DataRecord

pybiolib-1.1.1990/biolib/_internal/data_record/data_record.py

@@ -0,0 +1,166 @@
+import os
+from collections import namedtuple
+from datetime import datetime
+from fnmatch import fnmatch
+from struct import Struct
+from typing import Callable, Dict, List, Optional, Union, cast
+
+from biolib import lfs
+from biolib._internal.data_record.remote_storage_endpoint import DataRecordRemoteStorageEndpoint
+from biolib._internal.http_client import HttpClient
+from biolib.api import client as api_client
+from biolib.biolib_api_client import AppGetResponse
+from biolib.biolib_binary_format import LazyLoadedFile
+from biolib.biolib_binary_format.utils import RemoteIndexableBuffer
+from biolib.biolib_logging import logger
+from biolib.utils.app_uri import parse_app_uri
+from biolib.utils.zip.remote_zip import RemoteZip  # type: ignore
+
+PathFilter = Union[str, Callable[[str], bool]]
+
+
+class DataRecord:
+    def __init__(self, uri: str):
+        self._uri = uri
+        uri_parsed = parse_app_uri(uri, use_account_as_name_default=False)
+        if not uri_parsed['app_name']:
+            raise ValueError('Expected parameter "uri" to contain resource name')
+
+        self._name = uri_parsed['app_name']
+
+    def __repr__(self):
+        return f'DataRecord: {self._uri}'
+
+    @property
+    def uri(self) -> str:
+        return self._uri
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def list_files(self, path_filter: Optional[PathFilter] = None) -> List[LazyLoadedFile]:
+        app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': self._uri}).json()
+        remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
+            resource_version_uuid=app_response['app_version']['public_id'],
+        )
+        files: List[LazyLoadedFile] = []
+        with RemoteZip(url=remote_storage_endpoint.get_remote_url()) as remote_zip:
+            central_directory = remote_zip.get_central_directory()
+            for file_info in central_directory.values():
+                files.append(self._get_file(remote_storage_endpoint, file_info))
+
+        return self._get_filtered_files(files=files, path_filter=path_filter) if path_filter else files
+
+    def download_zip(self, output_path: str):
+        app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': self._uri}).json()
+        remote_storage_endpoint = DataRecordRemoteStorageEndpoint(
+            resource_version_uuid=app_response['app_version']['public_id'],
+        )
+        HttpClient.request(url=remote_storage_endpoint.get_remote_url(), response_path=output_path)
+
+    def download_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
+        filtered_files = self.list_files(path_filter=path_filter)
+
+        if len(filtered_files) == 0:
+            logger.debug('No files to save')
+            return
+
+        for file in filtered_files:
+            file_path = os.path.join(output_dir, file.path)
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            with open(file_path, mode='wb') as file_handle:
+                for chunk in file.get_data_iterator():
+                    file_handle.write(chunk)
+
+    def save_files(self, output_dir: str, path_filter: Optional[PathFilter] = None) -> None:
+        self.download_files(output_dir=output_dir, path_filter=path_filter)
+
+    @staticmethod
+    def create(destination: str, data_path: str, name: Optional[str] = None) -> 'DataRecord':
+        assert os.path.isdir(data_path), f'The path "{data_path}" is not a directory.'
+        record_name = name if name else 'data-record-' + datetime.now().isoformat().split('.')[0].replace(':', '-')
+        record_uri = lfs.create_large_file_system(lfs_uri=f'{destination}/{record_name}')
+        record_version_uri = lfs.push_large_file_system(lfs_uri=record_uri, input_dir=data_path)
+        return DataRecord(uri=record_version_uri)
+
+    @staticmethod
+    def fetch(uri: Optional[str] = None, count: Optional[int] = None) -> List['DataRecord']:
+        max_page_size = 1_000
+        params: Dict[str, Union[str, int]] = {
+            'page_size': str(count or max_page_size),
+            'resource_type': 'data-record',
+        }
+        if uri:
+            uri_parsed = parse_app_uri(uri, use_account_as_name_default=False)
+            params['account_handle'] = uri_parsed['account_handle_normalized']
+            if uri_parsed['app_name_normalized']:
+                params['app_name'] = uri_parsed['app_name_normalized']
+
+        results = api_client.get(path='/apps/', params=params).json()['results']
+        if count is None and len(results) == max_page_size:
+            logger.warning(
+                f'Fetch results exceeded maximum count of {max_page_size}. Some data records might not be fetched.'
+            )
+
+        return [DataRecord(result['resource_uri']) for result in results]
+
+    @staticmethod
+    def _get_file(remote_storage_endpoint: DataRecordRemoteStorageEndpoint, file_info: Dict) -> LazyLoadedFile:
+        local_file_header_signature_bytes = b'\x50\x4b\x03\x04'
+        local_file_header_struct = Struct('<H2sHHHIIIHH')
+        LocalFileHeader = namedtuple(
+            'LocalFileHeader',
+            (
+                'version',
+                'flags',
+                'compression_raw',
+                'mod_time',
+                'mod_date',
+                'crc_32_expected',
+                'compressed_size_raw',
+                'uncompressed_size_raw',
+                'file_name_len',
+                'extra_field_len',
+            ),
+        )
+
+        local_file_header_start = file_info['header_offset'] + len(local_file_header_signature_bytes)
+        local_file_header_end = local_file_header_start + local_file_header_struct.size
+
+        def file_start_func() -> int:
+            local_file_header_response = HttpClient.request(
+                url=remote_storage_endpoint.get_remote_url(),
+                headers={'range': f'bytes={local_file_header_start}-{local_file_header_end - 1}'},
+                timeout_in_seconds=300,
+            )
+            local_file_header = LocalFileHeader._make(
+                local_file_header_struct.unpack(local_file_header_response.content)
+            )
+            file_start: int = (
+                local_file_header_end + local_file_header.file_name_len + local_file_header.extra_field_len
+            )
+            return file_start
+
+        return LazyLoadedFile(
+            buffer=RemoteIndexableBuffer(endpoint=remote_storage_endpoint),
+            length=file_info['file_size'],
+            path=file_info['filename'],
+            start=None,
+            start_func=file_start_func,
+        )
+
+    @staticmethod
+    def _get_filtered_files(files: List[LazyLoadedFile], path_filter: PathFilter) -> List[LazyLoadedFile]:
+        if not (isinstance(path_filter, str) or callable(path_filter)):
+            raise Exception('Expected path_filter to be a string or a function')
+
+        if callable(path_filter):
+            return list(filter(lambda x: path_filter(x.path), files))  # type: ignore
+
+        glob_filter = cast(str, path_filter)
+
+        def _filter_function(file: LazyLoadedFile) -> bool:
+            return fnmatch(file.path, glob_filter)
+
+        return list(filter(_filter_function, files))

pybiolib-1.1.1990/biolib/_internal/data_record/remote_storage_endpoint.py

@@ -0,0 +1,27 @@
+from datetime import datetime, timedelta
+
+from biolib.api import client as api_client
+from biolib.biolib_api_client.lfs_types import LargeFileSystemVersion
+from biolib.biolib_binary_format.utils import RemoteEndpoint
+from biolib.biolib_logging import logger
+
+
+class DataRecordRemoteStorageEndpoint(RemoteEndpoint):
+    def __init__(self, resource_version_uuid: str):
+        self._resource_version_uuid: str = resource_version_uuid
+        self._expires_at = None
+        self._presigned_url = None
+
+    def get_remote_url(self):
+        if not self._presigned_url or datetime.utcnow() > self._expires_at:
+            lfs_version: LargeFileSystemVersion = api_client.get(
+                path=f'/lfs/versions/{self._resource_version_uuid}/',
+            ).json()
+            self._presigned_url = lfs_version['presigned_download_url']
+            self._expires_at = datetime.utcnow() + timedelta(minutes=8)
+            logger.debug(
+                f'DataRecord "{self._resource_version_uuid}" fetched presigned URL '
+                f'with expiry at {self._expires_at.isoformat()}'
+            )
+
+        return self._presigned_url

pybiolib-1.1.1990/biolib/_internal/fuse_mount/__init__.py

@@ -0,0 +1 @@
+from .experiment_fuse_mount import ExperimentFuseMount

pybiolib-1.1.1990/biolib/_internal/fuse_mount/experiment_fuse_mount.py

@@ -0,0 +1,209 @@
+import errno
+import os
+import stat
+from datetime import datetime, timezone
+from time import time
+
+from biolib._internal.libs.fusepy import FUSE, FuseOSError, Operations
+from biolib.biolib_errors import BioLibError
+from biolib.jobs import Job
+from biolib.typing_utils import Dict, List, Optional, Tuple, TypedDict
+
+
+class _AttributeDict(TypedDict):
+    st_atime: int
+    st_ctime: int
+    st_gid: int
+    st_mode: int
+    st_mtime: int
+    st_nlink: int
+    st_size: int
+    st_uid: int
+
+
+_SUCCESS_CODE = 0
+
+
+class ExperimentFuseMount(Operations):
+    def __init__(self, experiment):
+        self._experiment = experiment
+        self._job_names_map: Optional[Dict[str, Job]] = None
+        self._jobs_last_fetched_at: float = 0.0
+        self._mounted_at_epoch_seconds: int = int(time())
+
+    @staticmethod
+    def mount_experiment(experiment, mount_path: str) -> None:
+        FUSE(
+            operations=ExperimentFuseMount(experiment),
+            mountpoint=mount_path,
+            nothreads=True,
+            foreground=True,
+            allow_other=False,
+        )
+
+    def getattr(self, path: str, fh=None) -> _AttributeDict:
+        if path == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=self._mounted_at_epoch_seconds)
+
+        job, path_in_job = self._parse_path(path)
+        job_finished_at_epoch_seconds: int = int(
+            datetime.fromisoformat(job.to_dict()['finished_at'].rstrip('Z')).replace(tzinfo=timezone.utc).timestamp()
+        )
+
+        if path_in_job == '/':
+            return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        try:
+            file = job.get_output_file(path_in_job)
+            return self._get_file_attributes(
+                timestamp_epoch_seconds=job_finished_at_epoch_seconds,
+                size_in_bytes=file.length,
+            )
+        except BioLibError:
+            # file not found
+            pass
+
+        file_paths_in_job = [file.path for file in job.list_output_files()]
+
+        for file_path_in_job in file_paths_in_job:
+            if file_path_in_job.startswith(path_in_job):
+                return self._get_directory_attributes(timestamp_epoch_seconds=job_finished_at_epoch_seconds)
+
+        raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+    def readdir(self, path: str, fh: int) -> List[str]:
+        directory_entries = ['.', '..']
+
+        if path == '/':
+            directory_entries.extend(self._get_job_names_map(refresh_jobs=True).keys())
+        else:
+            job, path_in_job = self._parse_path(path)
+            dir_path_in_job = '/' if path_in_job == '/' else path_in_job + '/'
+            depth = dir_path_in_job.count('/')
+            directory_entries.extend(
+                set(
+                    [
+                        file.path.split('/')[depth]
+                        for file in job.list_output_files()
+                        if file.path.startswith(dir_path_in_job)
+                    ]
+                )
+            )
+
+        return directory_entries
+
+    def open(self, path: str, flags: int) -> int:
+        job, path_in_job = self._parse_path(path)
+        try:
+            job.get_output_file(path_in_job)
+        except BioLibError:
+            # file not found
+            raise FuseOSError(errno.ENOENT) from None
+
+        return 1234  # dummy file handle
+
+    def read(self, path: str, size: int, offset: int, fh: int) -> bytes:
+        job, path_in_job = self._parse_path(path)
+        try:
+            file = job.get_output_file(path_in_job)
+        except BioLibError:
+            raise FuseOSError(errno.ENOENT) from None  # No such file or directory
+
+        return file.get_data(start=offset, length=size)
+
+    def release(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def releasedir(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    def flush(self, path: str, fh: int) -> int:
+        return _SUCCESS_CODE
+
+    @staticmethod
+    def _get_directory_attributes(timestamp_epoch_seconds: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFDIR | 0o555,  # Directory that is readable and executable by owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=1,
+            st_uid=os.getuid(),
+        )
+
+    @staticmethod
+    def _get_file_attributes(timestamp_epoch_seconds: int, size_in_bytes: int) -> _AttributeDict:
+        return _AttributeDict(
+            st_atime=timestamp_epoch_seconds,
+            st_ctime=timestamp_epoch_seconds,
+            st_gid=os.getgid(),
+            st_mode=stat.S_IFREG | 0o444,  # Regular file with read permissions for owner, group, and others.
+            st_mtime=timestamp_epoch_seconds,
+            st_nlink=1,
+            st_size=size_in_bytes,
+            st_uid=os.getuid(),
+        )
+
+    def _get_job_names_map(self, refresh_jobs=False) -> Dict[str, Job]:
+        current_time = time()
+        if not self._job_names_map or (current_time - self._jobs_last_fetched_at > 1 and refresh_jobs):
+            self._jobs_last_fetched_at = current_time
+            self._job_names_map = {job.get_name(): job for job in self._experiment.get_jobs(status='completed')}
+
+        return self._job_names_map
+
+    def _parse_path(self, path: str) -> Tuple[Job, str]:
+        path_splitted = path.split('/')
+        job_name = path_splitted[1]
+        path_in_job = '/' + '/'.join(path_splitted[2:])
+        job = self._get_job_names_map().get(job_name)
+        if not job:
+            raise FuseOSError(errno.ENOENT)  # No such file or directory
+
+        return job, path_in_job
+
+    # ----------------------------------- File system methods not implemented below -----------------------------------
+
+    def chmod(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def chown(self, path, uid, gid):
+        raise FuseOSError(errno.EACCES)
+
+    def mknod(self, path, mode, dev):
+        raise FuseOSError(errno.EACCES)
+
+    def rmdir(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def mkdir(self, path, mode):
+        raise FuseOSError(errno.EACCES)
+
+    def unlink(self, path):
+        raise FuseOSError(errno.EACCES)
+
+    def symlink(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def rename(self, old, new):
+        raise FuseOSError(errno.EACCES)
+
+    def link(self, target, source):
+        raise FuseOSError(errno.EACCES)
+
+    def utimens(self, path, times=None):
+        raise FuseOSError(errno.EACCES)
+
+    def create(self, path, mode, fi=None):
+        raise FuseOSError(errno.EACCES)
+
+    def write(self, path, data, offset, fh):
+        raise FuseOSError(errno.EACCES)
+
+    def truncate(self, path, length, fh=None):
+        raise FuseOSError(errno.EACCES)
+
+    def fsync(self, path, datasync, fh):
+        raise FuseOSError(errno.EACCES)

{pybiolib-1.1.1730 → pybiolib-1.1.1990}/biolib/_internal/http_client.py

@@ -1,25 +1,25 @@
 import json
 import platform
-import time
+import shutil
 import socket
 import ssl
 import subprocess
-import urllib.request
+import time
 import urllib.error
 import urllib.parse
+import urllib.request
 
 from biolib.biolib_logging import logger_no_user_data
-from biolib.typing_utils import Dict, Optional, Union, Literal, cast
+from biolib.typing_utils import Dict, Literal, Optional, Union, cast
+
+_HttpMethod = Literal['GET', 'POST', 'PATCH', 'PUT']
 
 
 def _create_ssl_context():
     context = ssl.create_default_context()
     try:
         if platform.system() == 'Darwin':
-            certificates = subprocess.check_output(
-                "security find-certificate -a -p",
-                shell=True
-            ).decode('utf-8')
+            certificates = subprocess.check_output('security find-certificate -a -p', shell=True).decode('utf-8')
             context.load_verify_locations(cadata=certificates)
     except BaseException:
         pass
@@ -33,7 +33,7 @@ class HttpError(urllib.error.HTTPError):
             code=http_error.code,
             msg=http_error.msg,  # type: ignore
             hdrs=http_error.hdrs,  # type: ignore
-            fp=http_error.fp
+            fp=http_error.fp,
         )
 
     def __str__(self):
@@ -42,15 +42,24 @@ class HttpError(urllib.error.HTTPError):
 
 
 class HttpResponse:
-    def __init__(self, response):
+    def __init__(self, response, response_path) -> None:
         self.headers: Dict[str, str] = dict(response.headers)
         self.status_code: int = int(response.status)
-        self.content: bytes = response.read()
+        self.response_path = response_path
+        if self.response_path:
+            with open(self.response_path, 'wb') as out_file:
+                shutil.copyfileobj(response, out_file)
+        else:
+            self.content: bytes = response.read()
         self.url: str = response.geturl()
 
     @property
     def text(self) -> str:
-        return cast(str, self.content.decode('utf-8'))
+        if self.response_path:
+            with open(self.response_path, 'rb') as fp:
+                return cast(str, fp.read().decode('utf-8'))
+        else:
+            return cast(str, self.content.decode('utf-8'))
 
     def json(self):
         return json.loads(self.text)
@@ -61,12 +70,13 @@ class HttpClient:
 
     @staticmethod
     def request(
-            url: str,
-            method: Optional[Literal['GET', 'POST', 'PATCH', 'PUT']] = None,
-            data: Optional[Union[Dict, bytes]] = None,
-            headers: Optional[Dict[str, str]] = None,
-            retries: int = 5,
-            timeout_in_seconds: Optional[int] = None,
+        url: str,
+        method: Optional[_HttpMethod] = None,
+        data: Optional[Union[Dict, bytes]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        retries: int = 5,
+        timeout_in_seconds: Optional[int] = None,
+        response_path: Optional[str] = None,
     ) -> HttpResponse:
         if not HttpClient.ssl_context:
             HttpClient.ssl_context = _create_ssl_context()
@@ -84,7 +94,7 @@ class HttpClient:
         if timeout_in_seconds is None:
             timeout_in_seconds = 60 if isinstance(data, dict) else 180  # TODO: Calculate timeout based on data size
 
-        last_error: Optional[urllib.error.URLError] = None
+        last_error: Optional[Exception] = None
         for retry_count in range(retries + 1):
             if retry_count > 0:
                 time.sleep(5 * retry_count)
@@ -95,23 +105,32 @@ class HttpClient:
                     context=HttpClient.ssl_context,
                     timeout=timeout_in_seconds,
                 ) as response:
-                    return HttpResponse(response)
+                    return HttpResponse(response, response_path)
 
             except urllib.error.HTTPError as error:
-                if error.code == 502:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 502 for "{url}"')
+                if error.code == 429:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 429 for "{url}"')
+                    last_error = error
+                elif error.code == 502:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 502 for "{url}"')
                     last_error = error
                 elif error.code == 503:
-                    logger_no_user_data.debug(f'HTTP {method} request failed with status 503 for "{url}"')
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 503 for "{url}"')
+                    last_error = error
+                elif error.code == 504:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with status 504 for "{url}"')
                     last_error = error
                 else:
                     raise HttpError(error) from None
 
             except urllib.error.URLError as error:
                 if isinstance(error.reason, socket.timeout):
-                    logger_no_user_data.debug(f'HTTP {method} request failed with read timeout for "{url}"')
+                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                     last_error = error
                 else:
                     raise error
+            except socket.timeout as error:
+                logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                last_error = error
 
         raise last_error or Exception(f'HTTP {method} request failed after {retries} retries for "{url}"')

pybiolib-1.1.1990/biolib/_internal/libs/__init__.py

@@ -0,0 +1 @@
+# Note: this directory is purely for libraries to be directly included instead of as dependencies