pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/biolib_api_client/biolib_account_api.py

@@ -1,21 +0,0 @@
-import requests
-
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_errors import BioLibError
-
-
-class BiolibAccountApi:
-
-    @staticmethod
-    def fetch_by_handle(account_handle):
-        response = requests.get(
-            f'{BiolibApiClient.get().base_url}/api/account/{account_handle}',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            timeout=5
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        return response.json()

biolib/biolib_api_client/biolib_large_file_system_api.py

@@ -1,108 +0,0 @@
-import requests
-
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_api_client.lfs_types import LfsVersionPresignedUploadUrlResponse, LfsUploadPartMetadata, \
-    LargeFileSystemVersion, LargeFileSystem, ZipProxyFileListResponse
-from biolib.biolib_errors import BioLibError
-from biolib.typing_utils import List
-
-
-class BiolibLargeFileSystemApi:
-
-    @staticmethod
-    def create(account_uuid: str, name: str) -> LargeFileSystem:
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/lfs/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            timeout=5,
-            json={'account_uuid': account_uuid, 'name': name}
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        lfs: LargeFileSystem = response.json()
-        return lfs
-
-    @staticmethod
-    def fetch_version(lfs_version_uuid: str) -> LargeFileSystemVersion:
-        response = requests.get(
-            f'{BiolibApiClient.get().base_url}/api/lfs/versions/{lfs_version_uuid}/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            timeout=5,
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        lfs_version: LargeFileSystemVersion = response.json()
-        return lfs_version
-
-    @staticmethod
-    def fetch_file_list(presigned_download_url: str) -> ZipProxyFileListResponse:
-        base_url = BiolibApiClient.get().base_url
-        if base_url == 'https://biolib.com':
-            source_file_proxy_url = 'https://blbcdn.com/source-file-proxy'
-        elif base_url == 'https://staging.biolib.com':
-            source_file_proxy_url = 'https://staging.blbcdn.com/source-file-proxy'
-        else:
-            source_file_proxy_url = f'{base_url}/source-file-proxy'
-
-        response = requests.post(
-            f'{source_file_proxy_url}/files/',
-            timeout=10,
-            json={'zip_url': presigned_download_url},
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        response_dict: ZipProxyFileListResponse = response.json()
-        return response_dict
-
-    @staticmethod
-    def create_version(resource_uuid: str) -> LargeFileSystemVersion:
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/lfs/versions/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            json={'resource_uuid': resource_uuid},
-            timeout=5,
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        lfs_version: LargeFileSystemVersion = response.json()
-        return lfs_version
-
-    @staticmethod
-    def get_upload_url(resource_version_uuid: str, part_number: int) -> LfsVersionPresignedUploadUrlResponse:
-        response = requests.get(
-            f'{BiolibApiClient.get().base_url}/api/lfs/versions/{resource_version_uuid}/presigned_upload_url/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            params={'part_number': part_number},
-            timeout=5,
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())
-
-        presigned_upload_url_response: LfsVersionPresignedUploadUrlResponse = response.json()
-        return presigned_upload_url_response
-
-    @staticmethod
-    def complete_upload(
-            resource_version_uuid: str,
-            parts: List[LfsUploadPartMetadata],
-            size_bytes: int,
-    ) -> None:
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/lfs/versions/{resource_version_uuid}/complete_upload/',
-            auth=BearerAuth(BiolibApiClient.get().access_token),
-            json={'parts': parts, 'size_bytes': size_bytes},
-            timeout=5,
-        )
-
-        if not response.ok:
-            raise BioLibError(response.content.decode())

biolib/biolib_binary_format/aes_encrypted_package.py

@@ -1,42 +0,0 @@
-from Crypto.Cipher import AES
-
-from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
-
-
-class AesEncryptedPackage(BioLibBinaryFormatBasePackage):
-    def __init__(self, bbf=None):
-        super().__init__(bbf)
-        self.package_type = 7
-        self.iv_len = 12
-        self.tag_len = 16
-
-    def decrypt(self, aes_key_buffer):
-        iv, _, encrypted_data = self.deserialize()
-        aes_key = AES.new(aes_key_buffer, AES.MODE_GCM, iv)
-        return aes_key.decrypt(encrypted_data)
-
-    def serialize(self, iv, tag, encrypted_data):
-        bbf_data = bytearray()
-        bbf_data.extend(self.version.to_bytes(1, 'big'))
-        bbf_data.extend(self.package_type.to_bytes(1, 'big'))
-
-        bbf_data.extend(iv)
-        bbf_data.extend(tag)
-
-        bbf_data.extend(len(encrypted_data).to_bytes(8, 'big'))
-        bbf_data.extend(encrypted_data)
-
-        return bbf_data
-
-    def deserialize(self):
-        version = self.get_data(1, output_type='int')
-        package_type = self.get_data(1, output_type='int')
-        self.check_version_and_type(version=version, package_type=package_type, expected_package_type=self.package_type)
-
-        iv = self.get_data(self.iv_len)
-        tag = self.get_data(self.tag_len)
-
-        encrypted_data_len = self.get_data(8, output_type='int')
-        encrypted_data = self.get_data(encrypted_data_len)
-
-        return iv, tag, encrypted_data

biolib/biolib_binary_format/module_output.py

@@ -1,58 +0,0 @@
-from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
-
-
-class ModuleOutput(BioLibBinaryFormatBasePackage):
-    def __init__(self, bbf=None):
-        super().__init__(bbf)
-        self.package_type = 2
-
-    def serialize(self, stdout, stderr, exit_code, files):
-        bbf_data = bytearray()
-        bbf_data.extend(self.version.to_bytes(1, 'big'))
-        bbf_data.extend(self.package_type.to_bytes(1, 'big'))
-
-        # Length of stdout and stderr
-        bbf_data.extend(len(stdout).to_bytes(8, 'big'))
-        bbf_data.extend(len(stderr).to_bytes(8, 'big'))
-
-        file_data_len = sum([len(data) + len(path.encode()) for path, data in files.items()]) + (12 * len(files))
-        bbf_data.extend(file_data_len.to_bytes(8, 'big'))
-
-        bbf_data.extend(exit_code.to_bytes(2, 'big'))
-        bbf_data.extend(stdout)
-        bbf_data.extend(stderr)
-
-        for path, data in files.items():
-            # Path length and data length
-            bbf_data.extend(len(path.encode()).to_bytes(4, 'big'))
-            bbf_data.extend(len(data).to_bytes(8, 'big'))
-
-            bbf_data.extend(path.encode())
-            # The file data
-            bbf_data.extend(data)
-
-        return bbf_data
-
-    def deserialize(self):
-        version = self.get_data(1, output_type='int')
-        package_type = self.get_data(1, output_type='int')
-        self.check_version_and_type(version=version, package_type=package_type, expected_package_type=self.package_type)
-
-        stdout_len = self.get_data(8, output_type='int')
-        stderr_len = self.get_data(8, output_type='int')
-        files_data_len = self.get_data(8, output_type='int')
-        exit_code = self.get_data(2, output_type='int')
-        stdout = self.get_data(stdout_len)
-        stderr = self.get_data(stderr_len)
-
-        end_of_files = self.pointer + files_data_len
-        files = {}
-
-        while self.pointer < end_of_files:
-            path_len = self.get_data(4, output_type='int')
-            data_len = self.get_data(8, output_type='int')
-            path = self.get_data(path_len, output_type='str')
-            data = self.get_data(data_len)
-            files[path] = bytes(data)
-
-        return {'stdout': stdout, 'stderr': stderr, 'exit_code': exit_code, 'files': files}

biolib/biolib_binary_format/rsa_encrypted_aes_package.py

@@ -1,57 +0,0 @@
-from Crypto.Cipher import PKCS1_OAEP, AES
-from Crypto.PublicKey import RSA
-from Crypto.Random import get_random_bytes
-from Crypto.Hash import SHA256
-
-from biolib.biolib_binary_format.base_bbf_package import BioLibBinaryFormatBasePackage
-
-
-class RsaEncryptedAesPackage(BioLibBinaryFormatBasePackage):
-    def __init__(self, bbf=None):
-        super().__init__(bbf)
-        self.package_type = 6
-        self.iv_len = 12
-        self.aes_key_len = 32
-        self.tag_len = 16
-
-    def create(self, rsa_public_key_der, unencrypted_bytes):
-        public_key = RSA.import_key(rsa_public_key_der)
-        rsa_cipher = PKCS1_OAEP.new(public_key, hashAlgo=SHA256)
-
-        iv = get_random_bytes(self.iv_len)
-        aes_key_buffer = get_random_bytes(self.aes_key_len)
-        aes_key = AES.new(aes_key_buffer, AES.MODE_GCM, iv)
-        rsa_encrypted_aes_key_buffer = rsa_cipher.encrypt(aes_key_buffer)
-        aes_encrypted_data, tag = aes_key.encrypt_and_digest(bytes(unencrypted_bytes))
-
-        return self.serialize(rsa_encrypted_aes_key_buffer, iv, tag, aes_encrypted_data), aes_key_buffer
-
-    def serialize(self, encrypted_aes_key, iv, tag, encrypted_data):
-        bbf_data = bytearray()
-        bbf_data.extend(self.version.to_bytes(1, 'big'))
-        bbf_data.extend(self.package_type.to_bytes(1, 'big'))
-
-        bbf_data.extend(len(encrypted_aes_key).to_bytes(2, 'big'))
-        bbf_data.extend(encrypted_aes_key)
-        bbf_data.extend(iv)
-        bbf_data.extend(tag)
-
-        bbf_data.extend(len(encrypted_data).to_bytes(8, 'big'))
-        bbf_data.extend(encrypted_data)
-
-        return bbf_data
-
-    def deserialize(self):
-        version = self.get_data(1, output_type='int')
-        package_type = self.get_data(1, output_type='int')
-        self.check_version_and_type(version=version, package_type=package_type, expected_package_type=self.package_type)
-
-        encrypted_aes_key_len = self.get_data(2, output_type='int')
-        encrypted_aes_key = self.get_data(encrypted_aes_key_len)
-        iv = self.get_data(self.iv_len)
-        tag = self.get_data(self.tag_len)
-
-        encrypted_data_len = self.get_data(8, output_type='int')
-        encrypted_data = self.get_data(encrypted_data_len)
-
-        return encrypted_aes_key, iv, tag, encrypted_data

biolib/biolib_push.py

@@ -1,114 +0,0 @@
-from pathlib import Path
-import yaml
-
-from biolib import cli_utils
-from biolib.biolib_api_client import BiolibApiClient
-from biolib.biolib_docker_client import BiolibDockerClient
-from biolib.biolib_api_client.biolib_app_api import BiolibAppApi
-from biolib.biolib_errors import BioLibError
-from biolib.biolib_logging import logger
-from biolib.utils import get_absolute_container_image_uri
-
-
-def push_application(app_uri: str, app_path: str):
-    api_client = BiolibApiClient.get()
-    if not api_client.is_signed_in:
-        # TODO: Create an exception class for expected errors like this that does not print stacktrace
-        raise Exception(
-            'You must be authenticated to push an application.\n'
-            'Please set the environment variable "BIOLIB_TOKEN=[your_api_token]"\n'
-            'You can get an API token at: https://biolib.com/settings/api-tokens/'
-        ) from None
-
-    # prepare zip file
-    if not Path(f'{app_path}/.biolib/config.yml').is_file():
-        raise BioLibError('The file .biolib/config.yml was not found in the application directory')
-
-    tmp_directory, source_files_zip_path = cli_utils.create_temporary_directory_with_source_files_zip(app_path)
-
-    try:
-        with open(source_files_zip_path, mode='rb') as zip_file:
-            source_files_zip_bytes = zip_file.read()
-    except IOError:
-        raise Exception('Failed to read temporary source files zip') from None
-
-    try:
-        tmp_directory.cleanup()
-    except IOError:
-        raise Exception('Failed to clean up temporary source files zip directory') from None
-
-    # TODO: Raise exception if app_uri contains version
-    app_response = BiolibAppApi.get_by_uri(app_uri)
-    app = app_response['app']
-    # push new app version
-    new_app_version_json = BiolibAppApi.push_app_version(
-        app_id=app['public_id'],
-        app_name=app['name'],
-        author=app['account_handle'],
-        set_as_active=False,
-        zip_binary=source_files_zip_bytes,
-    )
-
-    docker_tags = new_app_version_json.get('docker_tags', {})
-    if docker_tags:
-        logger.info('Found docker images to push.')
-
-        try:
-            yaml_file = open(f'{app_path}/.biolib/config.yml', 'r', encoding='utf-8')
-
-        except Exception as error:  # pylint: disable=broad-except
-            raise BioLibError('Could not open the config file .biolib/config.yml') from error
-
-        try:
-            config_data = yaml.safe_load(yaml_file)
-
-        except Exception as error:  # pylint: disable=broad-except
-            raise BioLibError('Could not parse .biolib/config.yml. Please make sure it is valid YAML') from error
-
-        # Auth to be sent to proxy
-        # The tokens are sent as "{access_token},{job_id}". We leave job_id blank on push.
-        tokens = f'{BiolibApiClient.get().access_token},'
-        auth_config = {'username': 'biolib', 'password': tokens}
-
-        docker_client = BiolibDockerClient.get_docker_client()
-
-        for module_name, repo_and_tag in docker_tags.items():
-            docker_image_definition = config_data['modules'][module_name]['image']
-            repo, tag = repo_and_tag.split(':')
-
-            if docker_image_definition.startswith('dockerhub://'):
-                docker_image_name = docker_image_definition.replace('dockerhub://', 'docker.io/', 1)
-                logger.info(
-                    f'Pulling image {docker_image_name} defined on module {module_name} from Dockerhub.')
-                dockerhub_repo, dockerhub_tag = docker_image_name.split(':')
-                docker_client.images.pull(dockerhub_repo, tag=dockerhub_tag)
-
-            elif docker_image_definition.startswith('local-docker://'):
-                docker_image_name = docker_image_definition.replace('local-docker://', '', 1)
-
-            try:
-                logger.info(f'Trying to push image {docker_image_name} defined on module {module_name}.')
-                image = docker_client.images.get(docker_image_name)
-                image_uri = get_absolute_container_image_uri(
-                    base_url=api_client.base_url,
-                    relative_image_uri=repo
-                )
-                image.tag(image_uri, tag)
-                for line in docker_client.images.push(image_uri, tag=tag, stream=True,
-                                                      decode=True, auth_config=auth_config):
-                    logger.info(line)
-
-            except Exception as exception:  # pylint: disable=broad-except
-                raise BioLibError(f'Failed to tag and push image {docker_image_name}.') from exception
-
-            logger.info(f'Successfully pushed {docker_image_name}')
-
-        logger.info('Successfully pushed all docker images')
-
-    # Set new app version as active
-    BiolibAppApi.update_app_version(
-        app_version_id=new_app_version_json['public_id'],
-        data={
-            'set_as_active': True
-        }
-    )

biolib/cli.py

@@ -1,203 +0,0 @@
-import argparse
-import logging
-import os
-import sys
-import platform
-
-from docker.errors import ImageNotFound  # type: ignore
-
-from biolib import cli_utils, utils, biolib_errors
-from biolib.app import BioLibApp
-from biolib.app.utils import run_job
-from biolib.biolib_push import push_application
-from biolib.biolib_api_client import Job
-from biolib.biolib_binary_format import ModuleInput
-from biolib.cli_utils import BiolibValidationError
-from biolib.validators.validate_zip_file import validate_zip_file  # type: ignore
-from biolib.biolib_docker_client import BiolibDockerClient
-from biolib.biolib_logging import logger
-from biolib.lfs import create_large_file_system, push_large_file_system, describe_large_file_system
-
-
-class IllegalArgumentError(ValueError):
-    pass
-
-
-def file_path(path):
-    if path.startswith('/'):
-        full_path = path
-    else:
-        full_path = os.path.normpath(os.path.join(os.getcwd(), path))
-    if not os.path.exists(full_path):
-        raise IllegalArgumentError(f'The path {full_path} does not exist')
-    return full_path
-
-
-def port_number(port):
-    if not port.isdigit():
-        raise IllegalArgumentError(f'Port number {port} is not a number. Ports can only be numbers')
-
-    if not (0 < int(port) < 65000):  # pylint: disable=superfluous-parens
-        raise IllegalArgumentError('Port can only be between 0 and 65000')
-
-    return port
-
-
-def main():
-    # set more restrictive default log level for CLI
-    logger.configure(default_log_level=logging.WARNING)
-
-    def _get_stdin():
-        stdin = None
-        if not sys.stdin.isatty() and not utils.IS_DEV:
-            stdin = sys.stdin.read()
-        return stdin
-
-    if len(sys.argv) > 2 and sys.argv[1] == 'run':
-        try:
-            app = BioLibApp(uri=sys.argv[2])
-        except biolib_errors.BioLibError as error:
-            print(f'An error occurred:\n {error.message}', file=sys.stderr)
-            sys.exit(1)
-
-        app_args = sys.argv[3:]
-        result = app.cli(args=app_args, stdin=_get_stdin(), files=None)
-        result.save_files('biolib_results')
-        sys.stdout.buffer.write(result.stdout)
-        sys.stderr.buffer.write(result.stderr)
-        sys.exit(result.exitcode)
-
-    elif len(sys.argv) > 2 and sys.argv[1] == 'run-dev':
-        provided_app_path = sys.argv[2]
-        app_path = cli_utils.validate_and_get_app_path(provided_app_path)
-        logger.info(f'Running BioLib application in local directory {app_path}...')
-        tmp_directory, source_files_zip_path = cli_utils.create_temporary_directory_with_source_files_zip(app_path)
-
-        try:
-            validate_zip_file(source_files_zip_path, app_path)
-            yaml_data = cli_utils.get_yaml_data(app_path)
-            cli_utils.validate_yaml_config(yaml_data, source_files_zip_path)
-            app_version_mocked = cli_utils.get_mocked_app_version_from_yaml(yaml_data, source_files_zip_path)
-
-            for module in app_version_mocked['modules']:
-                try:
-                    BiolibDockerClient.get_docker_client().images.get(module['image_uri'])
-                except ImageNotFound:
-                    raise Exception(
-                        f"Could not find local docker image {module['image_uri']} specified in .biolib/config.yml"
-                    ) from None
-
-            app_args = sys.argv[3:]
-            args = cli_utils.validate_and_get_args(app_args)
-            files_dict, args = cli_utils.get_files_dict_and_file_args(files=None, args=args)
-
-            module_input_serialized = ModuleInput().serialize(
-                stdin=_get_stdin() or b'',
-                files=files_dict,
-                arguments=args,
-            )
-
-            module_output = run_job(
-                module_input_serialized=module_input_serialized,
-                job=Job(
-                    app_version=app_version_mocked,
-                    caller_job=None,
-                    created_at='',
-                    public_id=utils.RUN_DEV_JOB_ID,
-                    remote_hosts_with_warning=[],
-                    user_id=None,
-                ),
-            )
-
-            cli_utils.write_output_files(module_output['files'])
-
-            print(cli_utils.get_pretty_print_module_output_string(
-                exitcode=module_output['exit_code'],
-                stderr=module_output['stderr'],
-                stdout=module_output['stdout'],
-            ))
-
-        except Exception as exception:
-            # Exit on BiolibValidationError as we have already printed the validation errors
-            if isinstance(exception, BiolibValidationError):
-                logger.error('Validation check failed for config file at .biolib/config.yml')
-                sys.exit(1)
-            raise exception
-
-        finally:
-            try:
-                tmp_directory.cleanup()
-            except IOError:
-                logger.error('Failed to clean up temporary source files zip directory')
-
-    else:
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--version', action='version', version=utils.BIOLIB_PACKAGE_VERSION)
-
-        subparsers = parser.add_subparsers(help='command', dest='command')
-
-        # Add subparser for run to help message makes sense
-        # The actual code for running applications is above this
-        _parser_run = subparsers.add_parser('run', help='Run an application on BioLib')
-
-        # Add subparser for run to help message makes sense
-        # The actual code for running local applications is above this
-        _parser_run_dev = subparsers.add_parser('run-dev', help='Run an application from a local directory')
-
-        # add subparser for push
-        parser_push = subparsers.add_parser('push', help='Push an application to BioLib')
-        parser_push.add_argument('app_uri')
-        parser_push.add_argument('--path', default='.', required=False)
-
-        # add subparser for run-compute-node
-        parser_start = subparsers.add_parser('start', help='Start a compute node')
-        parser_start.add_argument('--port', default='5000', required=False, type=port_number)
-        parser_start.add_argument('--host', default='127.0.0.1', required=False)
-
-        # add subparser for lfs
-        parser_lfs = subparsers.add_parser('lfs', help='Work with large file systems')
-        subparsers_lfs = parser_lfs.add_subparsers(help='lfs command', dest='lfs_command')
-
-        parser_lfs_create = subparsers_lfs.add_parser('create', help='Create a large filesystem')
-        parser_lfs_create.add_argument('lfs_uri', type=str)
-
-        parser_lfs_push = subparsers_lfs.add_parser('push', help='Push data to a large filesystem')
-        parser_lfs_push.add_argument('lfs_uri', type=str)
-        parser_lfs_push.add_argument('--path', type=str)
-
-        parser_lfs_push = subparsers_lfs.add_parser('describe', help='Describe a large filesystem')
-        parser_lfs_push.add_argument('lfs_uri', type=str)
-        parser_lfs_push.add_argument('--json', action='store_true')
-
-        args = parser.parse_args()
-
-        if args.command == 'push':
-            logger.configure(default_log_level=logging.INFO)
-            push_application(args.app_uri, args.path)
-
-        elif args.command == 'start':
-            logger.configure(default_log_level=logging.INFO)
-            if platform.system() == 'Windows':
-                raise Exception('Starting a compute node is currently not supported on Windows')
-
-            from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
-            webserver.start_webserver(host=args.host, port=args.port)
-
-        elif args.command == 'lfs':
-            if args.lfs_command == 'create':
-                logger.configure(default_log_level=logging.INFO)
-                create_large_file_system(lfs_uri=args.lfs_uri)
-            elif args.lfs_command == 'push':
-                logger.configure(default_log_level=logging.INFO)
-                push_large_file_system(lfs_uri=args.lfs_uri, input_dir=args.path)
-            elif args.lfs_command == 'describe':
-                describe_large_file_system(lfs_uri=args.lfs_uri, output_as_json=args.json)
-
-        else:
-            print('Unrecognized command, please run biolib --help to see available options.')
-            sys.exit(1)
-
-
-# allow this script to be called without poetry in dev e.g. by an IDE debugger
-if utils.IS_DEV and __name__ == '__main__':
-    main()