pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/biolib_binary_format/utils.py

@@ -0,0 +1,175 @@
+import io
+import math
+import multiprocessing
+from abc import ABC, abstractmethod
+from multiprocessing.pool import ThreadPool
+from typing import Callable, Optional
+
+from biolib._internal.http_client import HttpClient
+from biolib.typing_utils import Iterator
+
+
+class RemoteEndpoint(ABC):
+    @abstractmethod
+    def get_remote_url(self):
+        pass
+
+
+class IndexableBuffer(ABC):
+    def __init__(self):
+        self.pointer = 0
+
+    @abstractmethod
+    def get_data(self, start: int, length: int) -> bytes:
+        pass
+
+    def get_data_as_string(self, start: int, length: int) -> str:
+        return self.get_data(start=start, length=length).decode()
+
+    def get_data_as_int(self, start: int, length: int) -> int:
+        return int.from_bytes(bytes=self.get_data(start=start, length=length), byteorder='big')
+
+    def get_data_with_pointer(self, length: int) -> bytes:
+        data = self.get_data(start=self.pointer, length=length)
+        self.pointer += length
+        return data
+
+    def get_data_with_pointer_as_int(self, length: int) -> int:
+        data = self.get_data_as_int(start=self.pointer, length=length)
+        self.pointer += length
+        return data
+
+    def get_data_with_pointer_as_string(self, length: int) -> str:
+        data = self.get_data_as_string(start=self.pointer, length=length)
+        self.pointer += length
+        return data
+
+
+class LocalFileIndexableBuffer(IndexableBuffer):
+    def __init__(self, filename: str):
+        super().__init__()
+        self._filehandle = open(filename, 'rb')
+
+    def get_data(self, start: int, length: int) -> bytes:
+        if length < 0:
+            raise Exception('get_data length must be positive')
+
+        if length == 0:
+            return bytes(0)
+
+        self._filehandle.seek(start)
+        data: bytes = self._filehandle.read(length)
+
+        if len(data) != length:
+            raise Exception(f'get_data got response of unexpected length. Got {len(data)} expected {length}.')
+
+        return data
+
+
+class RemoteIndexableBuffer(IndexableBuffer):
+    def __init__(self, endpoint: RemoteEndpoint):
+        super().__init__()
+        self._endpoint = endpoint
+
+    def get_data(self, start: int, length: int) -> bytes:
+        if length < 0:
+            raise Exception('get_data length must be positive')
+
+        if length == 0:
+            return bytes(0)
+
+        end = start + length - 1
+        response = HttpClient.request(
+            url=self._endpoint.get_remote_url(),
+            headers={'range': f'bytes={start}-{end}'},
+        )
+
+        data: bytes = response.content
+        if len(data) != length:
+            raise Exception(f'get_data got response of unexpected length. Got {len(data)} expected {length}.')
+
+        return data
+
+
+class InMemoryIndexableBuffer(IndexableBuffer):
+    def __init__(self, data: bytes):
+        super().__init__()
+        self._buffer = data
+        self._length_bytes = len(data)
+
+    def get_data(self, start: int, length: int) -> bytes:
+        end = start + length
+        return self._buffer[start:end]
+
+    def __len__(self):
+        return self._length_bytes
+
+
+class LazyLoadedFile:
+    def __init__(
+        self,
+        path: str,
+        buffer: IndexableBuffer,
+        start: Optional[int],
+        length: int,
+        start_func: Optional[Callable[[], int]] = None,
+    ):
+        self._path = path
+        self._buffer = buffer
+        self._start = start
+        self._start_func = start_func
+        self._length = length
+
+    def __repr__(self) -> str:
+        return f'File "{self._path}" with size of {self._length} bytes'
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    @property
+    def name(self) -> str:
+        return self._path.split('/')[-1]
+
+    @property
+    def start(self) -> int:
+        if self._start is None:
+            assert self._start_func is not None, 'No start function or start value'
+            self._start = self._start_func()
+
+        return self._start
+
+    @property
+    def length(self) -> int:
+        return self._length
+
+    def get_file_handle(self) -> io.BufferedIOBase:
+        return io.BytesIO(self.get_data())
+
+    def get_data(self, start=0, length=None) -> bytes:
+        start_offset = start + self.start
+        # make sure length doesn't go outside file boundaries
+        length_to_end_of_file = max(self._length - start, 0)
+        length_to_request = length_to_end_of_file if length is None else min(length, length_to_end_of_file)
+        return self._buffer.get_data(start=start_offset, length=length_to_request)
+
+    def get_data_iterator(self) -> Iterator[bytes]:
+        if self._length == 0:
+            yield b''
+        else:
+            chunk_size = 50_000_000
+            chunk_count = math.ceil(self._length / chunk_size)
+            chunk_indices_iterator = range(chunk_count - 1)
+
+            def get_chunk(chunk_index: int) -> bytes:
+                return self._buffer.get_data(start=self.start + chunk_index * chunk_size, length=chunk_size)
+
+            if chunk_count > 1:
+                with ThreadPool(processes=min(16, chunk_count, multiprocessing.cpu_count() - 1)) as pool:
+                    yield from pool.imap(func=get_chunk, iterable=chunk_indices_iterator)
+
+            data_already_yielded = (chunk_count - 1) * chunk_size
+            yield self._buffer.get_data(
+                start=self.start + data_already_yielded,
+                length=self._length - data_already_yielded,
+            )

biolib/biolib_docker_client/__init__.py

@@ -1,3 +1,5 @@
+import os
+
 import docker  # type: ignore
 
 
@@ -8,10 +10,17 @@ class BiolibDockerClient:
     def get_docker_client():
         if BiolibDockerClient.docker_client is None:
             try:
+                # Fixes: https://github.com/docker/docker-py/issues/2433
+                if os.environ.get('DOCKER_CERT_PATH'):
+                    request_ca_bundle_env = os.environ.pop('REQUESTS_CA_BUNDLE', None)
+
                 # the final step of docker push can take a long time,
                 # so set a long timeout for operations performed by the docker client
-                # 15 min (900s) is the maximum supported by AWS load balancers
-                BiolibDockerClient.docker_client = docker.from_env(timeout=900)
+                # ~66 min (4000s) is the maximum supported by AWS load balancers
+                BiolibDockerClient.docker_client = docker.from_env(timeout=4000)
+
+                if os.environ.get('DOCKER_CERT_PATH'):
+                    os.environ['REQUESTS_CA_BUNDLE'] = request_ca_bundle_env
                 # Run a docker command to see if docker engine is running
                 BiolibDockerClient.docker_client.info()
             except Exception as exception:

biolib/biolib_errors.py

@@ -7,6 +7,9 @@ class BioLibError(Exception):
         super().__init__(message)
         self.message = message
 
+class ValidationError(BioLibError):
+    def __init__(self, message: Optional[str] = None):
+        super().__init__(message=message or 'Invalid input.')
 
 class NotFound(BioLibError):
 
@@ -16,3 +19,36 @@ class NotFound(BioLibError):
 
 class DockerContainerNotFoundDuringExecutionException(Exception):
     pass
+
+
+class RetryLimitException(Exception):
+    pass
+
+
+class StorageDownloadFailed(Exception):
+    pass
+
+
+class CloudJobFinishedError(Exception):
+    pass
+
+
+class JobResultError(BioLibError):
+    pass
+
+
+class JobResultNotFound(JobResultError):
+    pass
+
+
+class JobResultPermissionError(JobResultError):
+    pass
+
+
+class JobResultNonZeroExitCodeError(JobResultError):
+    """Raised when an app returns a non-zero exit code and check=True."""
+    def __init__(self, exit_code: int, message: Optional[str] = None):
+        self.exit_code = exit_code
+        super().__init__(
+            message or f'App returned non-zero exit code: {exit_code}'
+        )

biolib/biolib_logging.py

@@ -1,9 +1,11 @@
 import logging
-import sys
 import os
+import sys
+
+_DEFAULT_LOGGER_FORMAT = '%(asctime)s | %(levelname)s : %(message)s'
 
 # define global logging format
-logging.basicConfig(format='%(asctime)s | %(levelname)s : %(message)s', level=logging.INFO, stream=sys.stdout)
+logging.basicConfig(format=_DEFAULT_LOGGER_FORMAT, level=logging.INFO, stream=sys.stdout)
 
 # define extra log levels
 TRACE = 5
@@ -12,7 +14,6 @@ logging.addLevelName(TRACE, 'TRACE')
 
 # note: Logger classes should never be instantiated directly
 class _BioLibLogger(logging.Logger):
-
     def __init__(self, name: str, level=logging.INFO):
         super(_BioLibLogger, self).__init__(name=name, level=level)
 
@@ -25,7 +26,8 @@ class _BioLibLogger(logging.Logger):
 
     def setLevel(self, level) -> None:
         try:
-            super(_BioLibLogger, self).setLevel(level)
+            normalized_level = level.upper() if isinstance(level, str) else level
+            super(_BioLibLogger, self).setLevel(normalized_level)
         except ValueError:
             raise Exception(f'Unknown log level "{level}"') from None
 
@@ -39,20 +41,35 @@ class _BioLibLogger(logging.Logger):
             global_root_logger.setLevel(self.level)
 
 
-def _get_biolib_logger_instance() -> _BioLibLogger:
+def _get_biolib_logger_instance(name: str) -> _BioLibLogger:
     # for thread safety use the global lock of logging
-    logging._acquireLock()  # type: ignore # pylint: disable=protected-access
+    logging._lock.acquire()  # type: ignore # pylint: disable=protected-access
+
     original_logger_class = logging.getLoggerClass()
     try:
         # change logger class temporarily to get instance of _BioLibLogger
         logging.setLoggerClass(_BioLibLogger)
-        biolib_logger = logging.getLogger('biolib')
+        biolib_logger = logging.getLogger(name=name)
         # change the logger class back to original so we do not interfere with other libraries
         logging.setLoggerClass(original_logger_class)
         return biolib_logger  # type: ignore
     finally:
-        logging._releaseLock()  # type: ignore # pylint: disable=protected-access
+        logging._lock.release()  # type: ignore # pylint: disable=protected-access
+
+
+def _get_no_user_data_logger() -> _BioLibLogger:
+    _logger_no_user_data = _get_biolib_logger_instance(name='biolib_no_user_data')
+
+    # TODO: Simplify by refactoring to env BIOLIB_ENVIRONMENT_IS_CLOUD: boolean
+    if os.getenv('BIOLIB_CLOUD_ENVIRONMENT', '').lower() == 'non-enclave':
+        handler = logging.FileHandler(filename='/biolib/logs/biolib_no_user_data.log')
+        formatter = logging.Formatter(_DEFAULT_LOGGER_FORMAT)
+        handler.setFormatter(formatter)
+        _logger_no_user_data.addHandler(handler)
+
+    return _logger_no_user_data
 
 
-# expose logger
-logger = _get_biolib_logger_instance()
+# expose loggers
+logger = _get_biolib_logger_instance(name='biolib')
+logger_no_user_data = _get_no_user_data_logger()

biolib/cli/__init__.py

@@ -0,0 +1,38 @@
+import logging
+import sys
+
+import click
+
+from biolib import utils
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.cli import auth, data_record, index, init, lfs, push, run, runtime, sdk, start
+
+
+@click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
+@click.group(context_settings=dict(help_option_names=['-h', '--help']))
+def cli() -> None:
+    logger_no_user_data.debug(f'pybiolib {utils.BIOLIB_PACKAGE_VERSION}')
+    logger_no_user_data.debug(f'Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')
+    utils.STREAM_STDOUT = True
+
+    # set more restrictive default log level for CLI
+    logger.configure(default_log_level=logging.WARNING)
+    logger_no_user_data.configure(default_log_level=logging.WARNING)
+
+
+cli.add_command(auth.login)
+cli.add_command(auth.logout)
+cli.add_command(auth.whoami)
+cli.add_command(init.init)
+cli.add_command(lfs.lfs)
+cli.add_command(push.push)
+cli.add_command(run.run)
+cli.add_command(runtime.runtime)
+cli.add_command(start.start)
+cli.add_command(data_record.data_record)
+cli.add_command(index.index)
+cli.add_command(sdk.sdk)
+
+# allow this script to be called without poetry in dev e.g. by an IDE debugger
+if utils.IS_DEV and __name__ == '__main__':
+    cli()

biolib/cli/auth.py

@@ -0,0 +1,46 @@
+import logging
+import sys
+
+import click
+
+from biolib import api
+from biolib.biolib_api_client.api_client import BiolibApiClient
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.user import sign_in, sign_out
+
+
+@click.command(help='Login your to BioLib account with web browser')
+@click.option(
+    '-w',
+    is_flag=True,
+    default=False,
+    required=False,
+    type=bool,
+    help='Automatically open the login page in the default web browser',
+)
+def login(w: bool) -> None:  # pylint: disable=invalid-name
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_in(open_in_default_browser=w)
+
+
+@click.command(help='Logout of your BioLib account')
+def logout() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_out()
+
+
+@click.command(help='Prints out the full name of the user logged in')
+def whoami() -> None:
+    client = BiolibApiClient.get()
+    if client.is_signed_in:
+        response = api.client.get(path='/users/me/')
+        user_dict = response.json()
+        email = user_dict['email']
+        display_name = user_dict['account']['display_name']
+
+        print(f'Name: {display_name}\nEmail: {email}\nLogged into: {client.base_url}')
+    else:
+        print('Not logged in', file=sys.stderr)
+        exit(1)

biolib/cli/data_record.py

@@ -0,0 +1,164 @@
+import json
+import logging
+import os
+import sys
+from typing import Dict, List
+
+import click
+import rich.progress
+
+from biolib._data_record.data_record import DataRecord
+from biolib.biolib_api_client import BiolibApiClient
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.typing_utils import Optional
+
+
+@click.group(help='Data Records')
+def data_record() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@data_record.command(help='Create a Data Record')
+@click.argument('uri', required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+@click.option('--record-type', required=False, type=str, default=None)
+def create(uri: str, data_path: str, record_type: Optional[str]) -> None:
+    DataRecord.create(destination=uri, data_path=data_path, record_type=record_type)
+
+
+@data_record.command(help='Update a Data Record')
+@click.argument('uri', required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+@click.option('--chunk-size', default=None, required=False, type=click.INT, help='The size of each chunk (In MB)')
+def update(uri: str, data_path: str, chunk_size: Optional[int]) -> None:
+    DataRecord.get_by_uri(uri=uri).update(data_path=data_path, chunk_size_in_mb=chunk_size)
+
+
+@data_record.command(help='Download files from a Data Record')
+@click.argument('uri', required=True)
+@click.option('--file', required=False, type=str)
+@click.option('--path-filter', required=False, type=str, hide_input=True)
+def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+    if file is not None:
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
+    else:
+        assert not os.path.exists(record.name), f'Directory with name {record.name} already exists in current directory'
+        record.save_files(output_dir=record.name, path_filter=path_filter)
+
+
+@data_record.command(help='Describe a Data Record')
+@click.argument('uri', required=True)
+@click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
+def describe(uri: str, output_as_json: bool) -> None:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='get Data Record description')
+    record = DataRecord.get_by_uri(uri)
+    files_info: List[Dict] = []
+    total_size_in_bytes = 0
+    for file in record.list_files():
+        files_info.append({'path': file.path, 'size_bytes': file.length})
+        total_size_in_bytes += file.length
+
+    if output_as_json:
+        print(
+            json.dumps(
+                obj={'uri': record.uri, 'size_bytes': total_size_in_bytes, 'files': files_info},
+                indent=4,
+            )
+        )
+    else:
+        print(f'Data Record {record.uri}\ntotal {total_size_in_bytes} bytes\n')
+        print('size bytes    path')
+        for file_info in files_info:
+            size_string = str(file_info['size_bytes'])
+            leading_space_string = ' ' * (10 - len(size_string))
+            print(f"{leading_space_string}{size_string}    {file_info['path']}")
+
+
+@data_record.command(help='Delete a Data Record')
+@click.argument('uri', required=True)
+def delete(uri: str) -> None:
+    record = DataRecord.get_by_uri(uri=uri)
+
+    print(f'You are about to delete the data record: {record.uri}')
+    print('This action cannot be undone.')
+
+    confirmation = input(f'To confirm deletion, please type the data record name "{record.name}": ')
+    if confirmation != record.name:
+        print('Data record name does not match. Deletion cancelled.')
+        return
+
+    record.delete()
+    print(f'Data record {record.uri} has been deleted.')
+
+
+def _clone_data_record_with_progress(
+    source_record: DataRecord,
+    dest_record: DataRecord,
+) -> None:
+    # pylint: disable=protected-access
+    total_size_in_bytes = source_record._get_zip_size_bytes()
+    # pylint: enable=protected-access
+
+    if total_size_in_bytes == 0:
+        logger.info('Source data record has no data to clone')
+        return
+
+    if sys.stdout.isatty():
+        with rich.progress.Progress(
+            rich.progress.TextColumn('[bold blue]{task.description}'),
+            rich.progress.BarColumn(),
+            rich.progress.TaskProgressColumn(),
+            rich.progress.TimeRemainingColumn(),
+            rich.progress.TransferSpeedColumn(),
+        ) as progress:
+            task_id = progress.add_task('Cloning data record', total=total_size_in_bytes)
+
+            def on_progress(bytes_uploaded: int, _total_bytes: int) -> None:
+                progress.update(task_id, completed=bytes_uploaded)
+
+            DataRecord.clone(source=source_record, destination=dest_record, on_progress=on_progress)
+    else:
+        logger.info(f'Cloning ~{round(total_size_in_bytes / 10**6)}mb of data')
+        DataRecord.clone(source=source_record, destination=dest_record)
+
+
+def _get_or_create_destination_record(destination_uri: str) -> Optional[DataRecord]:
+    try:
+        return DataRecord.get_by_uri(uri=destination_uri)
+    except Exception:
+        print(f'Destination data record "{destination_uri}" does not exist.')
+        confirmation = input('Would you like to create it? [y/N]: ')
+        if confirmation.lower() != 'y':
+            print('Clone cancelled.')
+            return None
+
+        return DataRecord.create(destination=destination_uri)
+
+
+@data_record.command(help='Clone a Data Record to another location')
+@click.argument('source_uri', required=True)
+@click.argument('destination_uri', required=True)
+def clone(source_uri: str, destination_uri: str) -> None:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='clone a Data Record')
+
+    logger.info(f'Fetching source data record: {source_uri}')
+    source_record = DataRecord.get_by_uri(uri=source_uri)
+
+    logger.info(f'Checking destination data record: {destination_uri}')
+    dest_record = _get_or_create_destination_record(destination_uri)
+    if dest_record is None:
+        return
+
+    logger.info(f'Cloning from {source_record.uri} to {dest_record.uri}...')
+    _clone_data_record_with_progress(source_record=source_record, dest_record=dest_record)
+    logger.info('Clone completed successfully.')

biolib/cli/index.py

@@ -0,0 +1,32 @@
+import json
+import logging
+import sys
+
+import click
+
+from biolib._index.index import Index
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger, logger_no_user_data
+
+
+@click.group(help='Manage Indexes')
+def index() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@index.command(help='Create an Index')
+@click.argument('uri', required=True)
+@click.option('--config-path', required=True, type=click.Path(exists=True), help='Path to JSON config file')
+def create(uri: str, config_path: str) -> None:
+    try:
+        Index.create_from_config_file(uri=uri, config_path=config_path)
+    except json.JSONDecodeError as error:
+        print(f'Error: Invalid JSON in config file: {error}', file=sys.stderr)
+        sys.exit(1)
+    except BioLibError as error:
+        print(f'Error creating index: {error.message}', file=sys.stderr)
+        sys.exit(1)
+    except Exception as error:
+        print(f'Error reading config file: {error}', file=sys.stderr)
+        sys.exit(1)