pybiolib 1.1.1747__py3-none-any.whl → 1.1.1881__py3-none-any.whl

biolib/biolib_binary_format/utils.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 import io
+from typing import Optional, Callable
 
 from biolib._internal.http_client import HttpClient
 
@@ -106,10 +107,18 @@ class InMemoryIndexableBuffer(IndexableBuffer):
 
 class LazyLoadedFile:
 
-    def __init__(self, path: str, buffer: IndexableBuffer, start: int, length: int):
+    def __init__(
+            self,
+            path: str,
+            buffer: IndexableBuffer,
+            start: Optional[int],
+            length: int,
+            start_func: Optional[Callable[[], int]] = None,
+    ):
         self._path = path
         self._buffer = buffer
         self._start = start
+        self._start_func = start_func
         self._length = length
 
     def __repr__(self) -> str:
@@ -119,8 +128,16 @@ class LazyLoadedFile:
     def path(self) -> str:
         return self._path
 
+    @property
+    def name(self) -> str:
+        return self._path.split('/')[-1]
+
     @property
     def start(self) -> int:
+        if self._start is None:
+            assert self._start_func is not None, 'No start function or start value'
+            self._start = self._start_func()
+
         return self._start
 
     @property
@@ -131,4 +148,4 @@ class LazyLoadedFile:
         return io.BytesIO(self.get_data())
 
     def get_data(self) -> bytes:
-        return self._buffer.get_data(start=self._start, length=self._length)
+        return self._buffer.get_data(start=self.start, length=self._length)

biolib/cli/__init__.py

@@ -5,7 +5,7 @@ import click
 
 from biolib import utils
 from biolib.biolib_logging import logger, logger_no_user_data
-from biolib.cli import init, lfs, push, run, start, runtime, download_container
+from biolib.cli import auth, data_record, download_container, init, lfs, push, run, runtime, start
 
 
 @click.version_option(version=utils.BIOLIB_PACKAGE_VERSION, prog_name='pybiolib')
@@ -20,13 +20,17 @@ def cli() -> None:
     logger_no_user_data.configure(default_log_level=logging.WARNING)
 
 
+cli.add_command(auth.login)
+cli.add_command(auth.logout)
+cli.add_command(auth.whoami)
+cli.add_command(download_container.download_container)
 cli.add_command(init.init)
 cli.add_command(lfs.lfs)
 cli.add_command(push.push)
 cli.add_command(run.run)
 cli.add_command(runtime.runtime)
 cli.add_command(start.start)
-cli.add_command(download_container.download_container)
+cli.add_command(data_record.data_record)
 
 # allow this script to be called without poetry in dev e.g. by an IDE debugger
 if utils.IS_DEV and __name__ == '__main__':

biolib/cli/auth.py

@@ -0,0 +1,58 @@
+import logging
+import sys
+
+import click
+
+from biolib import api, biolib_errors
+from biolib.biolib_api_client.api_client import BiolibApiClient
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.user import sign_in, sign_out
+
+
+@click.command(help='Login your to BioLib account with web browser')
+@click.option(
+    '-w',
+    is_flag=True,
+    default=False,
+    required=False,
+    type=bool,
+    help='Automatically open the login page in the default web browser',
+)
+def login(w: bool) -> None:  # pylint: disable=invalid-name
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_in(open_in_default_browser=w)
+
+
+@click.command(help='Logout of your BioLib account')
+def logout() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+    sign_out()
+
+
+@click.command(help='Prints out the full name of the user logged in')
+def whoami() -> None:
+    client = BiolibApiClient.get()
+    if client.is_signed_in:
+        user_uuid = None
+        if client.access_token is None:
+            print('Unable to fetch user credentials. Please try logging out and logging in again.')
+            exit(1)
+        try:
+            user_uuid = client.decode_jwt_without_checking_signature(jwt=client.access_token)['payload']['public_id']
+        except biolib_errors.BioLibError as error:
+            print(
+                f'Unable to reference user public_id in access token:\n {error.message}',
+                file=sys.stderr,
+            )
+            exit(1)
+        response = api.client.get(path=f'/user/{user_uuid}/')
+        user_dict = response.json()
+        email = user_dict['email']
+        intrinsic_account = [account for account in user_dict['accounts'] if account['role'] == 'intrinsic'][0]
+        display_name = intrinsic_account['display_name']
+        print(f'Name: {display_name}\nEmail: {email}')
+    else:
+        print('Not logged in', file=sys.stderr)
+        exit(1)

biolib/cli/data_record.py

@@ -0,0 +1,43 @@
+import logging
+import os
+
+import click
+
+from biolib._internal.data_record import DataRecord
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.typing_utils import Optional
+
+
+@click.group(help='Data Records')
+def data_record() -> None:
+    logger.configure(default_log_level=logging.INFO)
+    logger_no_user_data.configure(default_log_level=logging.INFO)
+
+
+@data_record.command(help='Create a Data Record')
+@click.option('--destination', type=str, required=True)
+@click.option('--data-path', required=True, type=click.Path(exists=True))
+@click.option('--name', type=str, required=False)
+def create(destination: str, data_path: str, name: Optional[str] = None) -> None:
+    DataRecord.create(destination, data_path, name)
+
+
+@data_record.command(help='Download files from a Data Record')
+@click.argument('uri', required=True)
+@click.option('--file', required=False, type=str)
+@click.option('--path-filter', required=False, type=str, hide_input=True)
+def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
+    record = DataRecord(uri=uri)
+    if file is not None:
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
+    else:
+        assert not os.path.exists(record.name), f'Directory with name {record.name} already exists in current directory'
+        record.save_files(output_dir=record.name, path_filter=path_filter)

biolib/cli/download_container.py

@@ -1,10 +1,12 @@
 import logging
+
 import click
+
 from biolib.biolib_download_container import download_container_from_uri
 from biolib.biolib_logging import logger, logger_no_user_data
 
 
-@click.command(help='Push an application to BioLib', name='download-container')
+@click.command(help='Pull an application from BioLib', name='download-container', hidden=True)
 @click.argument('uri')
 def download_container(uri: str) -> None:
     logger.configure(default_log_level=logging.INFO)

biolib/cli/init.py

@@ -2,6 +2,7 @@ import os
 import sys
 
 import click
+
 from biolib import templates
 
 

biolib/cli/lfs.py

@@ -1,12 +1,15 @@
+import json
 import logging
+import os
 import sys
+from typing import Dict, List
 
 import click
 
-import biolib.lfs
 from biolib import biolib_errors
-from biolib.biolib_logging import logger_no_user_data, logger
-from biolib.lfs import push_large_file_system, create_large_file_system, describe_large_file_system, prune_lfs_cache
+from biolib._internal.data_record import DataRecord
+from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.lfs import create_large_file_system, prune_lfs_cache, push_large_file_system
 from biolib.typing_utils import Optional
 
 
@@ -44,9 +47,16 @@ def download_file(uri: str, file_path: str) -> None:
     logger.configure(default_log_level=logging.INFO)
     logger_no_user_data.configure(default_log_level=logging.INFO)
     try:
-        data = biolib.lfs.get_file_data_from_large_file_system(lfs_uri=uri, file_path=file_path)
-        with open(file_path, mode='wb') as file:
-            file.write(data)
+        record = DataRecord(uri=uri)
+        try:
+            file_obj = [file_obj for file_obj in record.list_files() if file_obj.path == file_path][0]
+        except IndexError:
+            raise Exception('File not found in data record') from None
+
+        assert not os.path.exists(file_obj.name), 'File already exists in current directory'
+        with open(file_obj.name, 'wb') as file_handle:
+            file_handle.write(file_obj.get_data())
+
     except biolib_errors.BioLibError as error:
         print(f'An error occurred:\n{error.message}', file=sys.stderr)
         exit(1)
@@ -54,9 +64,29 @@ def download_file(uri: str, file_path: str) -> None:
 
 @lfs.command(help='Describe a Large File System')
 @click.argument('uri', required=True)
-@click.option('--json', is_flag=True, default=False, required=False, help='Format output as JSON')
-def describe(uri: str, json: bool) -> None:
-    describe_large_file_system(lfs_uri=uri, output_as_json=json)
+@click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
+def describe(uri: str, output_as_json: bool) -> None:
+    data_record = DataRecord(uri)
+    files_info: List[Dict] = []
+    total_size_in_bytes = 0
+    for file in data_record.list_files():
+        files_info.append({'path': file.path, 'size_bytes': file.length})
+        total_size_in_bytes += file.length
+
+    if output_as_json:
+        print(
+            json.dumps(
+                obj={'uri': data_record.uri, 'size_bytes': total_size_in_bytes, 'files': files_info},
+                indent=4,
+            )
+        )
+    else:
+        print(f'Large File System {data_record.uri}\ntotal {total_size_in_bytes} bytes\n')
+        print('size bytes    path')
+        for file_info in files_info:
+            size_string = str(file_info['size_bytes'])
+            leading_space_string = ' ' * (10 - len(size_string))
+            print(f"{leading_space_string}{size_string}    {file_info['path']}")
 
 
 @lfs.command(help='Prune LFS cache', hidden=True)

biolib/cli/push.py

@@ -3,8 +3,8 @@ from typing import Optional
 
 import click
 
-from biolib.biolib_logging import logger, logger_no_user_data
 from biolib._internal.push_application import push_application
+from biolib.biolib_logging import logger, logger_no_user_data
 
 
 @click.command(help='Push an application to BioLib')

biolib/cli/run.py

@@ -1,4 +1,5 @@
 import sys
+
 import click
 
 from biolib import biolib_errors, utils
@@ -27,13 +28,13 @@ def run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
             stdin = sys.stdin.read()
         return stdin
 
-    blocking = False if non_blocking else True
+    blocking = not non_blocking
     job = app.cli(
         args=list(args),
         stdin=_get_stdin(),
         files=None,
         machine=('local' if local else ''),
-        blocking=blocking
+        blocking=blocking,
     )
 
     if blocking:

biolib/cli/start.py

@@ -24,6 +24,7 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
 
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
+
         webserver.start_webserver(
             host=host,
             port=port,

biolib/compute_node/cloud_utils/cloud_utils.py

@@ -81,8 +81,8 @@ class CloudUtils:
                         'error': error,
                     },
                 )
-            except BaseException as error:
-                logger_no_user_data.error(f'Failed to deregister got error: {error}')
+            except BaseException as error_object:
+                logger_no_user_data.error(f'Failed to deregister got error: {error_object}')
         else:
             logger_no_user_data.error("Not deregistering as environment is not cloud")
 

biolib/compute_node/job_worker/cache_state.py

@@ -16,7 +16,7 @@ class DockerCacheStateError(CacheStateError):
 
 class LfsCacheState(CacheState):
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
 
         self._storage_path_for_write: str = self._get_storage_path_for_write()

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -18,6 +18,8 @@ from docker.errors import ImageNotFound, APIError  # type: ignore
 from docker.models.containers import Container  # type: ignore
 
 from biolib import utils
+
+from biolib._internal.runtime import RuntimeJobDataDict
 from biolib.biolib_binary_format import ModuleInput, ModuleOutputV2
 from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_errors import DockerContainerNotFoundDuringExecutionException, BioLibError
@@ -37,7 +39,7 @@ from biolib.typing_utils import List, Dict, Optional
 
 class DockerExecutor:
 
-    def __init__(self, options: LocalExecutorOptions):
+    def __init__(self, options: LocalExecutorOptions) -> None:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
@@ -267,12 +269,12 @@ class DockerExecutor:
             internal_network = self._options['internal_network']
             extra_hosts: Dict[str, str] = {}
 
-            biolib_system_secret = {
-                'version': '1.0.0',
-                'job_requested_machine': self._options['job']['requested_machine'],
-                'job_uuid': self._options['job']['public_id'],
-                'job_auth_token': self._options['job']['auth_token'],
-            }
+            biolib_system_secret = RuntimeJobDataDict(
+                version='1.0.0',
+                job_requested_machine=self._options['job']['requested_machine'],
+                job_uuid=self._options['job']['public_id'],
+                job_auth_token=self._options['job']['auth_token'],
+            )
             secrets: Dict[str, str] = dict(
                 **module.get('secrets', {}),
                 biolib_system_secret=json.dumps(biolib_system_secret, indent=4),

biolib/compute_node/job_worker/job_worker.py

@@ -97,7 +97,11 @@ class JobWorker:
         if socket_port:
             self._connect_to_parent()
 
-    def _handle_exit_gracefully(self, signum: int, frame: FrameType) -> None:  # pylint: disable=unused-argument
+    def _handle_exit_gracefully(
+            self,
+            signum: int,
+            frame: Optional[FrameType],  # pylint: disable=unused-argument
+    ) -> None:
         job_id = self._root_job_wrapper["job"]["public_id"] if self._root_job_wrapper else None
         logger_no_user_data.debug(
             f'_JobWorker ({job_id}) got exit signal {signal.Signals(signum).name}'  # pylint: disable=no-member
@@ -246,6 +250,7 @@ class JobWorker:
 
     def _start_network_and_remote_host_proxies(self, job: CreatedJobDict) -> None:
         app_version = job['app_version']
+        app = app_version.get('app', {})
         job_id = job['public_id']
         remote_hosts = app_version['remote_hosts']
         if utils.IS_RUNNING_IN_CLOUD:
@@ -307,7 +312,8 @@ class JobWorker:
                         self._public_network,
                         self._internal_network,
                         job_id,
-                        ports
+                        ports,
+                        can_push_data_record_for_user=app.get('can_push_data_record_for_user', False),
                     )
                     remote_host_proxy.start()
                     self._remote_host_proxies.append(remote_host_proxy)

biolib/compute_node/remote_host_proxy.py

@@ -31,8 +31,10 @@ class RemoteHostProxy:
             public_network: Network,
             internal_network: Optional[Network],
             job_id: str,
-            ports: List[int]
+            ports: List[int],
+            can_push_data_record_for_user: bool,
     ):
+        self._can_push_data_record_for_user: bool = can_push_data_record_for_user
         self.is_app_caller_proxy = remote_host['hostname'] == 'AppCallerProxy'
 
         # Default to port 443 for now until backend serves remote_hosts with port specified
@@ -165,7 +167,7 @@ class RemoteHostProxy:
 
             if utils.IS_RUNNING_IN_CLOUD:
                 config = CloudUtils.get_webserver_config()
-                s3_results_bucket_name = config['s3_general_storage_bucket_name']  # pylint: disable=unsubscriptable-object
+                s3_results_bucket_name = config['s3_general_storage_bucket_name']
                 s3_results_base_url = f'https://{s3_results_bucket_name}.s3.amazonaws.com'
             else:
                 if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
@@ -205,6 +207,12 @@ http {{
         default    "";
     }}
 
+    map $request_method $bearer_token_on_post_and_get {{
+        POST       "{bearer_token}";
+        GET        "{bearer_token}";
+        default    "";
+    }}
+
     server {{
         listen       80;
         resolver 127.0.0.11 valid=30s;
@@ -279,6 +287,13 @@ http {{
             proxy_ssl_server_name    on;
         }}
 
+        location /api/lfs/ {{
+            proxy_pass               {base_url}/api/lfs/;
+            proxy_set_header         authorization {'$bearer_token_on_post_and_get' if self._can_push_data_record_for_user else '""'};
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
         location /api/ {{
             proxy_pass               {base_url}/api/;
             proxy_set_header         authorization "";
@@ -307,12 +322,25 @@ http {{
             proxy_ssl_server_name    on;
         }}
 
+        {f"""
+        location /proxy/storage/lfs/versions/ {{
+            proxy_pass               {cloud_base_url}/proxy/storage/lfs/versions/;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+        """ if self._can_push_data_record_for_user else ''}
+
         location /proxy/cloud/ {{
             proxy_pass               {cloud_base_url}/proxy/cloud/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
+
+        location / {{
+            return 404 "Not found";
+        }}
     }}
 }}
 '''

biolib/jobs/job.py

@@ -8,6 +8,8 @@ from urllib.parse import urlparse
 
 from biolib import api, utils
 from biolib._internal.http_client import HttpClient
+from biolib._internal.utils import open_browser_window_from_notebook
+from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import LazyLoadedFile, ModuleOutputV2, ModuleInput, ModuleInputDict
 from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
@@ -23,12 +25,14 @@ from biolib.utils import IS_RUNNING_IN_NOTEBOOK
 
 class Job:
     # Columns to print in table when showing Job
-    table_columns_to_row_map = OrderedDict({
-        'ID': {'key': 'uuid', 'params': {'width': 36}},
-        'Application': {'key': 'app_uri', 'params': {}},
-        'Status': {'key': 'state', 'params': {}},
-        'Started At': {'key': 'started_at', 'params': {}},
-    })
+    table_columns_to_row_map = OrderedDict(
+        {
+            'ID': {'key': 'uuid', 'params': {'width': 36}},
+            'Application': {'key': 'app_uri', 'params': {}},
+            'Status': {'key': 'state', 'params': {}},
+            'Started At': {'key': 'started_at', 'params': {}},
+        }
+    )
 
     def __init__(self, job_dict: JobDict):
         self._uuid: str = job_dict['uuid']
@@ -172,6 +176,17 @@ class Job:
             time.sleep(2)
         logger.info(f'Job {self.id} has finished.')
 
+    def open_browser(self) -> None:
+        api_client = BiolibApiClient.get()
+        results_url_to_open = f'{api_client.base_url}/results/{self.id}/?token={self._auth_token}'
+        if IS_RUNNING_IN_NOTEBOOK:
+            print(f'Opening results page at: {results_url_to_open}')
+            print('If your browser does not open automatically, click on the link above.')
+            open_browser_window_from_notebook(results_url_to_open)
+        else:
+            print('Please copy and paste the following link into your browser:')
+            print(results_url_to_open)
+
     def _get_cloud_job(self) -> CloudJobDict:
         self._refetch_job_dict(force_refetch=True)
         if self._job_dict['cloud_job'] is None:
@@ -190,20 +205,11 @@ class Job:
     @staticmethod
     def show_jobs(count: int = 25) -> None:
         job_dicts = Job._get_job_dicts(count)
-        BioLibTable(
-            columns_to_row_map=Job.table_columns_to_row_map,
-            rows=job_dicts,
-            title='Jobs'
-        ).print_table()
+        BioLibTable(columns_to_row_map=Job.table_columns_to_row_map, rows=job_dicts, title='Jobs').print_table()
 
     @staticmethod
     def _get_job_dicts(count: int) -> List['JobDict']:
-        job_dicts: List['JobDict'] = api.client.get(
-            path='/jobs/',
-            params={
-                'page_size': str(count)
-            }
-        ).json()['results']
+        job_dicts: List['JobDict'] = api.client.get(path='/jobs/', params={'page_size': str(count)}).json()['results']
         return job_dicts
 
     @staticmethod
@@ -235,9 +241,7 @@ class Job:
     def show(self) -> None:
         self._refetch_job_dict()
         BioLibTable(
-            columns_to_row_map=Job.table_columns_to_row_map,
-            rows=[self._job_dict],
-            title=f'Job: {self._uuid}'
+            columns_to_row_map=Job.table_columns_to_row_map, rows=[self._job_dict], title=f'Job: {self._uuid}'
         ).print_table()
 
     def stream_logs(self) -> None:
@@ -305,12 +309,10 @@ class Job:
 
     def _print_full_logs(self, node_url: str) -> None:
         try:
-            response_json = HttpClient.request(
-                url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full'
-            ).json()
+            response_json = HttpClient.request(url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full').json()
         except Exception as error:
             logger.error(f'Could not get full streamed logs due to: {error}')
-            raise BioLibError from error
+            raise BioLibError('Could not get full streamed logs') from error
 
         for status_update in response_json.get('previous_status_updates', []):
             logger.info(f'Cloud: {status_update["log_message"]}')
@@ -336,9 +338,7 @@ class Job:
     def _get_job_status_from_compute_node(self, compute_node_url):
         for _ in range(15):
             try:
-                return HttpClient.request(
-                    url=f'{compute_node_url}/v1/job/{self._uuid}/status/'
-                ).json()
+                return HttpClient.request(url=f'{compute_node_url}/v1/job/{self._uuid}/status/').json()
             except Exception:  # pylint: disable=broad-except
                 cloud_job = self._get_cloud_job()
                 logger.debug("Failed to get status from compute node, retrying...")
@@ -346,8 +346,7 @@ class Job:
                     logger.debug("Job no longer exists on compute node, checking for error...")
                     if cloud_job['error_code'] != SystemExceptionCodes.COMPLETED_SUCCESSFULLY.value:
                         error_message = SystemExceptionCodeMap.get(
-                            cloud_job['error_code'],
-                            f'Unknown error code {cloud_job["error_code"]}'
+                            cloud_job['error_code'], f'Unknown error code {cloud_job["error_code"]}'
                         )
                         raise BioLibError(f'Cloud: {error_message}') from None
                     else:

biolib/lfs/__init__.py

@@ -1,6 +1,4 @@
 from .cache import prune_lfs_cache
 from .utils import \
-    describe_large_file_system, \
-    get_file_data_from_large_file_system, \
     push_large_file_system, \
     create_large_file_system