pybiolib 1.1.1629__py3-none-any.whl → 1.1.1881__py3-none-any.whl

biolib/app/search_apps.py

@@ -7,41 +7,37 @@ from biolib.typing_utils import Optional, List
 def search_apps(
         search_query: Optional[str] = None,
         team: Optional[str] = None,
-        count: int = 100
-    ) -> List[str]:
-
+        count: int = 100,
+) -> List[str]:
     query_exceeded_page_size = False
     params = {
         'page_size': count,
     }
     if team:
-        if not team.startswith("@"):
-            team = "@biolib.com/" + team
+        if not team.startswith('@'):
+            team = '@biolib.com/' + team
         params['account_handle'] = team
 
     if search_query:
         params['search'] = search_query
 
-    apps_json = api.client.get(
-        path='/apps/',
-        params=params
-    ).json()
+    apps_json = api.client.get(path='/apps/', params=params).json()
     if apps_json['count'] > count:
         query_exceeded_page_size = True
 
     apps = [app['resource_uri'] for app in apps_json['results']]
 
-    if not utils.BASE_URL_IS_PUBLIC_BIOLIB and (not team or team.lower().startswith("@biolib.com")):
+    if not utils.BASE_URL_IS_PUBLIC_BIOLIB and (not team or team.lower().startswith('@biolib.com')):
         # Also get federated apps if running on enterprise deployment
         public_biolib_apps_json = api.client.get(
             authenticate=False,
             path='https://biolib.com/api/apps/',
-            params=params
+            params=params,
         ).json()
         if public_biolib_apps_json['count'] > count:
             query_exceeded_page_size = True
 
-        apps.extend([f'@biolib.com/{app["resource_uri"]}' for app in public_biolib_apps_json['results']])
+        apps.extend([f"@biolib.com/{app['resource_uri']}" for app in public_biolib_apps_json['results']])
 
     if query_exceeded_page_size:
         print(f'Search results exceeded {count}, use the argument "count" to increase the amount of results returned')

biolib/biolib_api_client/api_client.py

@@ -6,6 +6,7 @@ import os
 from datetime import datetime, timezone
 from json.decoder import JSONDecodeError
 
+from biolib._internal.runtime import Runtime
 from biolib._internal.http_client import HttpClient
 from biolib.typing_utils import Optional
 from biolib.biolib_errors import BioLibError
@@ -61,16 +62,18 @@ class _ApiClient:
             return
 
         if self.access_token:
-            decoded_token = self._decode_jwt_without_checking_signature(self.access_token)
+            decoded_token = self.decode_jwt_without_checking_signature(self.access_token)
             if datetime.now(tz=timezone.utc).timestamp() < decoded_token['payload']['exp'] - 60:  # 60 second buffer
                 # Token has not expired yet
                 return
 
         # TODO: Implement nicer error handling
         try:
-            response = HttpClient.request(method='POST',
-                                        url=f'{self.base_url}/api/user/token/refresh/',
-                                        data={'refresh': self.refresh_token})
+            response = HttpClient.request(
+                method='POST',
+                url=f'{self.base_url}/api/user/token/refresh/',
+                data={'refresh': self.refresh_token},
+            )
         except Exception as exception:
             logger.error('Sign in with refresh token failed')
             raise exception
@@ -111,9 +114,11 @@ class _ApiClient:
     def sign_in_with_api_token(self, api_token: str) -> None:
         logger_no_user_data.debug('ApiClient: Signing in with BIOLIB_TOKEN...')
         try:
-            response = HttpClient.request(method='POST',
-                                        url=f'{self.base_url}/api/user/api_tokens/exchange/',
-                                        data={'token': api_token})
+            response = HttpClient.request(
+                method='POST',
+                url=f'{self.base_url}/api/user/api_tokens/exchange/',
+                data={'token': api_token},
+            )
         except Exception as exception:
             logger.error('Sign in with API token failed')
             raise exception
@@ -127,7 +132,7 @@ class _ApiClient:
         self.refresh_token = json_response['refresh_token']
 
     @staticmethod
-    def _decode_jwt_without_checking_signature(jwt: str) -> Dict[str, Any]:
+    def decode_jwt_without_checking_signature(jwt: str) -> Dict[str, Any]:
         jwt_bytes = jwt.encode('utf-8')
 
         try:
@@ -189,7 +194,7 @@ class BiolibApiClient:
     @staticmethod
     def assert_is_signed_in(authenticated_action_description: str) -> None:
         api_client = BiolibApiClient.get()
-        if not api_client.is_signed_in:
+        if not api_client.is_signed_in and not Runtime.check_is_environment_biolib_app():
             raise BioLibError(
                 f'You must be signed in to {authenticated_action_description}. '
                 f'Please set the environment variable "BIOLIB_TOKEN"'

biolib/biolib_api_client/app_types.py

@@ -99,6 +99,7 @@ class _AppVersionOnJob(TypedDict):
 
 class AppOnJob(TypedDict):
     allow_client_side_execution: bool
+    can_push_data_record_for_user: bool
     state: Literal['public', 'draft']
 
 

biolib/biolib_api_client/auth.py

@@ -1,20 +1,8 @@
-from requests.auth import AuthBase  # type: ignore
-
 from biolib import api
 from biolib.biolib_api_client.api_client import UserTokens
 from biolib.typing_utils import TypedDict, Literal
 
 
-class BearerAuth(AuthBase):
-    def __init__(self, access_token=None):
-        self.access_token = access_token
-
-    def __call__(self, req):
-        if self.access_token:
-            req.headers['Authorization'] = 'Bearer ' + self.access_token
-        return req
-
-
 class AuthChallengeCreate(TypedDict):
     token: str
 

biolib/biolib_api_client/biolib_app_api.py

@@ -1,18 +1,43 @@
+import mimetypes
+import random
 import re
 import os
 import subprocess
 
-import requests
-
 import biolib.api
 from biolib import biolib_errors
+from biolib._internal.http_client import HttpError
 from biolib.typing_utils import Optional
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient, AppGetResponse
-from biolib.biolib_errors import BioLibError
+from biolib.biolib_api_client import AppGetResponse
 from biolib.biolib_logging import logger
 
 
+def encode_multipart(data, files):
+    boundary = f'----------{random.randint(0, 1000000000)}'
+    line_array = []
+
+    for (key, value) in data.items():
+        if not value is None:
+            line_array.append(f'--{boundary}')
+            line_array.append(f'Content-Disposition: form-data; name="{key}"')
+            line_array.append('')
+            line_array.append(value)
+
+    for (key, (filename, value)) in files.items():
+        line_array.append(f'--{boundary}')
+        line_array.append(f'Content-Disposition: form-data; name="{key}"; filename="{filename}"')
+        line_array.append(f'Content-Type: {mimetypes.guess_type(filename)[0] or "application/octet-stream"}')
+        line_array.append('')
+        line_array.append('')
+        line_array.append(value)
+
+    line_array.append(f'--{boundary}--')
+    line_array.append('')
+
+    data_encoded = b'\r\n'.join([line.encode() if isinstance(line, str) else line for line in line_array])
+    return f'multipart/form-data; boundary={boundary}', data_encoded
+
+
 def _get_git_branch_name() -> str:
     try:
         github_actions_branch_name = os.getenv('GITHUB_REF_NAME')
@@ -51,13 +76,10 @@ class BiolibAppApi:
             app_response: AppGetResponse = response.json()
             return app_response
 
-        except requests.exceptions.HTTPError as error:
-            if error.response.status_code == 404:
+        except HttpError as error:
+            if error.code == 404:
                 raise biolib_errors.NotFound(f'Application {uri} not found.') from None
 
-            if error.response.status_code == 400:
-                raise biolib_errors.BioLibError(error.response.content.decode()) from None
-
             raise error
 
     @staticmethod
@@ -69,24 +91,28 @@ class BiolibAppApi:
             set_as_active,
             app_version_id_to_copy_images_from: Optional[str],
     ):
-        response = requests.post(
-            f'{BiolibApiClient.get().base_url}/api/app_versions/',
-            files={
-                'source_files_zip': zip_binary,
-            },
-            data={
-                'app': app_id,
-                'set_as_active': 'true' if set_as_active else 'false',
-                'state': 'published',
-                'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
-                'git_branch_name': _get_git_branch_name(),
-                'git_repository_url': _get_git_repository_url(),
-            },
-            auth=BearerAuth(BiolibApiClient.get().access_token)
-        )
-        if not response.ok:
+        try:
+            content_type, data_encoded = encode_multipart(
+                data={
+                    'app': app_id,
+                    'set_as_active': 'true' if set_as_active else 'false',
+                    'state': 'published',
+                    'app_version_id_to_copy_images_from': app_version_id_to_copy_images_from,
+                    'git_branch_name': _get_git_branch_name(),
+                    'git_repository_url': _get_git_repository_url(),
+                },
+                files={
+                    'source_files_zip': ('source_files.zip', zip_binary),
+                }
+            )
+            response = biolib.api.client.post(
+                path='/app_versions/',
+                data=data_encoded,
+                headers={'Content-Type': content_type},
+            )
+        except Exception as error:
             logger.error(f'Push failed for {author}/{app_name}:')
-            raise BioLibError(response.text)
+            raise error
 
         # TODO: When response includes the version number, print the URL for the new app version
         logger.info(f'Initialized new app version for {author}/{app_name}.')

biolib/biolib_api_client/biolib_job_api.py

@@ -1,16 +1,12 @@
 import os
-import time
 from urllib.parse import urlparse
 
-import requests
-
 import biolib.api
 
 from biolib import utils
-from biolib.biolib_api_client.auth import BearerAuth
-from biolib.biolib_api_client import BiolibApiClient, CloudJob, JobState
-from biolib.biolib_errors import BioLibError, RetryLimitException, StorageDownloadFailed, JobResultPermissionError, \
-    JobResultError, JobResultNotFound
+from biolib._internal.http_client import HttpError
+from biolib.biolib_api_client import CloudJob, JobState
+from biolib.biolib_errors import JobResultPermissionError, JobResultError, JobResultNotFound, StorageDownloadFailed
 from biolib.biolib_logging import logger
 from biolib.utils import BIOLIB_PACKAGE_VERSION
 from biolib.typing_utils import TypedDict, Optional, Literal, Dict
@@ -96,35 +92,11 @@ class BiolibJobApi:
 
     @staticmethod
     def create_cloud_job(job_id: str, result_name_prefix: Optional[str]) -> CloudJob:
-        response = None
         data = {'job_id': job_id}
         if result_name_prefix:
             data['result_name_prefix'] = result_name_prefix
 
-        for retry in range(4):
-            try:
-                response = requests.post(
-                    f'{BiolibApiClient.get().base_url}/api/jobs/cloud/',
-                    json=data,
-                    auth=BearerAuth(BiolibApiClient.get().access_token)
-                )
-
-                if response.status_code == 503:
-                    raise RetryLimitException(response.content)
-                # Handle possible validation errors from backend
-                elif not response.ok:
-                    raise BioLibError(response.text)
-
-                break
-
-            except RetryLimitException as retry_exception:  # pylint: disable=broad-except
-                if retry > 3:
-                    raise BioLibError('Reached retry limit for cloud job creation') from retry_exception
-                time.sleep(1)
-
-        if not response:
-            raise BioLibError('Could not create new cloud job')
-
+        response = biolib.api.client.post(path='/jobs/cloud/', data=data)
         cloud_job: CloudJob = response.json()
         return cloud_job
 
@@ -136,9 +108,9 @@ class BiolibJobApi:
     ) -> str:
         try:
             response = biolib.api.client.get(
-                path=f'{BiolibApiClient.get().base_url}/api/jobs/{job_uuid}/storage/{storage_type}/download/',
+                path=f'/jobs/{job_uuid}/storage/{storage_type}/download/',
                 authenticate=True,
-                headers={'Job-Auth-Token': job_auth_token}
+                headers={'Job-Auth-Token': job_auth_token},
             )
             presigned_s3_download_link_response: PresignedS3DownloadLinkResponse = response.json()
             presigned_download_url = presigned_s3_download_link_response['presigned_download_url']
@@ -151,21 +123,20 @@ class BiolibJobApi:
 
             return presigned_download_url
 
-        except requests.exceptions.HTTPError as error:
-            status_code = error.response.status_code
+        except HttpError as error:
             if storage_type == 'results':
-                if status_code == 401:
+                if error.code == 401:
                     raise JobResultPermissionError('You must be signed in to get result of the job') from None
-                elif status_code == 403:
+                elif error.code == 403:
                     raise JobResultPermissionError(
                         'Cannot get result of job. Maybe the job was created without being signed in?'
                     ) from None
-                elif status_code == 404:
+                elif error.code == 404:
                     raise JobResultNotFound('Job result not found') from None
                 else:
                     raise JobResultError('Failed to get result of job') from error
             else:
-                raise StorageDownloadFailed(error.response.content) from error
+                raise StorageDownloadFailed(f'Failed to download result of job got error: {error}') from error
 
         except Exception as error:  # pylint: disable=broad-except
             if storage_type == 'results':

biolib/biolib_binary_format/utils.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 import io
+from typing import Optional, Callable
 
 from biolib._internal.http_client import HttpClient
 
@@ -106,10 +107,18 @@ class InMemoryIndexableBuffer(IndexableBuffer):
 
 class LazyLoadedFile:
 
-    def __init__(self, path: str, buffer: IndexableBuffer, start: int, length: int):
+    def __init__(
+            self,
+            path: str,
+            buffer: IndexableBuffer,
+            start: Optional[int],
+            length: int,
+            start_func: Optional[Callable[[], int]] = None,
+    ):
         self._path = path
         self._buffer = buffer
         self._start = start
+        self._start_func = start_func
         self._length = length
 
     def __repr__(self) -> str:
@@ -119,8 +128,16 @@ class LazyLoadedFile:
     def path(self) -> str:
         return self._path
 
+    @property
+    def name(self) -> str:
+        return self._path.split('/')[-1]
+
     @property
     def start(self) -> int:
+        if self._start is None:
+            assert self._start_func is not None, 'No start function or start value'
+            self._start = self._start_func()
+
         return self._start
 
     @property
@@ -131,4 +148,4 @@ class LazyLoadedFile:
         return io.BytesIO(self.get_data())
 
     def get_data(self) -> bytes:
-        return self._buffer.get_data(start=self._start, length=self._length)
+        return self._buffer.get_data(start=self.start, length=self._length)

biolib/cli/__init__.py

@@ -1,16 +1,18 @@
 import logging
+import sys
 
 import click
 
 from biolib import utils
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.cli import init, lfs, push, run, start, runtime, download_container
+from biolib.cli import auth, data_record, download_container, init, lfs, push, run, runtime, start
 
 
 @click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
 @click.group(context_settings=dict(help_option_names=['-h', '--help']))
 def cli() -> None:
-    logger_no_user_data.debug(f'pybiolib, version {utils.BIOLIB_PACKAGE_VERSION}')
+    logger_no_user_data.debug(f'pybiolib {utils.BIOLIB_PACKAGE_VERSION}')
+    logger_no_user_data.debug(f'Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')
     utils.STREAM_STDOUT = True
 
     # set more restrictive default log level for CLI
@@ -18,13 +20,17 @@ def cli() -> None:
     logger_no_user_data.configure(default_log_level=logging.WARNING)
 
 
+cli.add_command(auth.login)
+cli.add_command(auth.logout)
+cli.add_command(auth.whoami)
+cli.add_command(download_container.download_container)
 cli.add_command(init.init)
 cli.add_command(lfs.lfs)
 cli.add_command(push.push)
 cli.add_command(run.run)
 cli.add_command(runtime.runtime)
 cli.add_command(start.start)
-cli.add_command(download_container.download_container)
+cli.add_command(data_record.data_record)
 
 # allow this script to be called without poetry in dev e.g. by an IDE debugger
 if utils.IS_DEV and __name__ == '__main__':

biolib/cli/auth.py

@@ -0,0 +1,58 @@
+import logging
+import sys
+
+import click
+
+from biolib import api, biolib_errors
+from biolib.biolib_api_client.api_client import BiolibApiClient
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.user import sign_in, sign_out
+
+
+@click.command(help='Login your to BioLib account with web browser')
+@click.option(
+    '-w',
+    is_flag=True,
+    default=False,
+    required=False,
+    type=bool,
+    help='Automatically open the login page in the default web browser',
+)
+def login(w: bool) -> None:  # pylint: disable=invalid-name
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_in(open_in_default_browser=w)
+
+
+@click.command(help='Logout of your BioLib account')
+def logout() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_out()
+
+
+@click.command(help='Prints out the full name of the user logged in')
+def whoami() -> None:
+    client = BiolibApiClient.get()
+    if client.is_signed_in:
+        user_uuid = None
+        if client.access_token is None:
+            print('Unable to fetch user credentials. Please try logging out and logging in again.')
+            exit(1)
+        try:
+            user_uuid = client.decode_jwt_without_checking_signature(jwt=client.access_token)['payload']['public_id']
+        except biolib_errors.BioLibError as error:
+            print(
+                f'Unable to reference user public_id in access token:\n {error.message}',
+                file=sys.stderr,
+            )
+            exit(1)
+        response = api.client.get(path=f'/user/{user_uuid}/')
+        user_dict = response.json()
+        email = user_dict['email']
+        intrinsic_account = [account for account in user_dict['accounts'] if account['role'] == 'intrinsic'][0]
+        display_name = intrinsic_account['display_name']
+        print(f'Name: {display_name}\nEmail: {email}')
+    else:
+        print('Not logged in', file=sys.stderr)
+        exit(1)

biolib/cli/data_record.py

@@ -0,0 +1,43 @@
+import logging
+import os
+
+import click
+
+from biolib._internal.data_record import DataRecord
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.typing_utils import Optional
+
+
+@click.group(help='Data Records')
+def data_record() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@data_record.command(help='Create a Data Record')
+@click.option('--destination', type=str, required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+@click.option('--name', type=str, required=False)
+def create(destination: str, data_path: str, name: Optional[str] = None) -> None:
+    DataRecord.create(destination, data_path, name)
+
+
+@data_record.command(help='Download files from a Data Record')
+@click.argument('uri', required=True)
+@click.option('--file', required=False, type=str)
+@click.option('--path-filter', required=False, type=str, hide_input=True)
+def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
+    record = DataRecord(uri=uri)
+    if file is not None:
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
+    else:
+        assert not os.path.exists(record.name), f'Directory with name {record.name} already exists in current directory'
+        record.save_files(output_dir=record.name, path_filter=path_filter)

biolib/cli/download_container.py

@@ -1,10 +1,12 @@
 import logging
+
 import click
+
 from biolib.biolib_download_container import download_container_from_uri
 from biolib.biolib_logging import logger, logger_no_user_data
 
 
-@click.command(help='Push an application to BioLib', name='download-container')
+@click.command(help='Pull an application from BioLib', name='download-container', hidden=True)
 @click.argument('uri')
 def download_container(uri: str) -> None:
     logger.configure(default_log_level=logging.INFO)

biolib/cli/init.py

@@ -2,6 +2,7 @@ import os
 import sys
 
 import click
+
 from biolib import templates
 
 

biolib/cli/lfs.py

@@ -1,12 +1,15 @@
+import json
 import logging
+import os
 import sys
+from typing import Dict, List
 
 import click
 
-import biolib.lfs
 from biolib import biolib_errors
-from biolib.biolib_logging import logger_no_user_data, logger
-from biolib.lfs import push_large_file_system, create_large_file_system, describe_large_file_system, prune_lfs_cache
+from biolib._internal.data_record import DataRecord
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.lfs import create_large_file_system, prune_lfs_cache, push_large_file_system
 from biolib.typing_utils import Optional
 
 
@@ -44,9 +47,16 @@ def download_file(uri: str, file_path: str) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     try:
-        data = biolib.lfs.get_file_data_from_large_file_system(lfs_uri=uri, file_path=file_path)
-        with open(file_path, mode='wb') as file:
-            file.write(data)
+        record = DataRecord(uri=uri)
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file_path][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
     except biolib_errors.BioLibError as error:
         print(f'An error occurred:\n{error.message}', file=sys.stderr)
         exit(1)
@@ -54,9 +64,29 @@ def download_file(uri: str, file_path: str) -> None:
 
 @lfs.command(help='Describe a Large File System')
 @click.argument('uri', required=True)
-@click.option('--json', is_flag=True, default=False, required=False, help='Format output as JSON')
-def describe(uri: str, json: bool) -> None:
-    describe_large_file_system(lfs_uri=uri, output_as_json=json)
+@click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
+def describe(uri: str, output_as_json: bool) -> None:
+    data_record = DataRecord(uri)
+    files_info: List[Dict] = []
+    total_size_in_bytes = 0
+    for file in data_record.list_files():
+        files_info.append({'path': file.path, 'size_bytes': file.length})
+        total_size_in_bytes += file.length
+
+    if output_as_json:
+        print(
+            json.dumps(
+                obj={'uri': data_record.uri, 'size_bytes': total_size_in_bytes, 'files': files_info},
+                indent=4,
+            )
+        )
+    else:
+        print(f'Large File System {data_record.uri}\ntotal {total_size_in_bytes} bytes\n')
+        print('size bytes    path')
+        for file_info in files_info:
+            size_string = str(file_info['size_bytes'])
+            leading_space_string = ' ' * (10 - len(size_string))
+            print(f"{leading_space_string}{size_string}    {file_info['path']}")
 
 
 @lfs.command(help='Prune LFS cache', hidden=True)

biolib/cli/push.py

@@ -3,8 +3,8 @@ from typing import Optional
 
 import click
 
-from biolib.biolib_logging import logger, logger_no_user_data
 from biolib._internal.push_application import push_application
+from biolib.biolib_logging import logger, logger_no_user_data
 
 
 @click.command(help='Push an application to BioLib')