pybiolib 1.1.1629__py3-none-any.whl → 1.1.1881__py3-none-any.whl

biolib/cli/run.py

@@ -1,4 +1,5 @@
 import sys
+
 import click
 
 from biolib import biolib_errors, utils
@@ -27,13 +28,13 @@ def run(local: bool, non_blocking: bool, uri: str, args: Tuple[str]) -> None:
             stdin = sys.stdin.read()
         return stdin
 
-    blocking = False if non_blocking else True
+    blocking = not non_blocking
     job = app.cli(
         args=list(args),
         stdin=_get_stdin(),
         files=None,
         machine=('local' if local else ''),
-        blocking=blocking
+        blocking=blocking,
     )
 
     if blocking:

biolib/cli/start.py

@@ -24,6 +24,7 @@ def start(host: str, port: int, tls_certificate: Optional[str], tls_key: Optiona
 
     try:
         from biolib.compute_node.webserver import webserver  # pylint: disable=import-outside-toplevel
+
         webserver.start_webserver(
             host=host,
             port=port,

biolib/compute_node/cloud_utils/cloud_utils.py

@@ -6,12 +6,10 @@ import subprocess
 import time
 from datetime import datetime
 from socket import gethostbyname, gethostname
-import requests
 
-from biolib import utils
-from biolib.biolib_errors import BioLibError
+from biolib import utils, api
 from biolib.biolib_logging import logger_no_user_data
-from biolib.typing_utils import Optional, List
+from biolib.typing_utils import Optional, List, Dict, cast
 from biolib.biolib_api_client import BiolibApiClient
 from biolib.compute_node.webserver.webserver_types import WebserverConfig, ComputeNodeInfo, ShutdownTimes
 
@@ -70,23 +68,21 @@ class CloudUtils:
         return CloudUtils._webserver_config
 
     @staticmethod
-    def deregister() -> None:
+    def deregister(error: Optional[str] = None) -> None:
         if utils.IS_RUNNING_IN_CLOUD:
             config = CloudUtils.get_webserver_config()
             try:
-                response = requests.post(url=f'{config["base_url"]}/api/jobs/deregister/', json={
-                    'auth_token': config["compute_node_info"]["auth_token"],
-                    'public_id': config["compute_node_info"]["public_id"],
-                })
-
-                if not response.ok:
-                    response_content = response.content.decode()
-                    logger_no_user_data.error(
-                        f'Failed to deregister got status {response.status_code} and response {response_content}'
-                    )
-
-            except Exception as error:  # pylint: disable=broad-except
-                logger_no_user_data.error(f'Failed to deregister got error: {error}')
+                api.client.post(
+                    authenticate=False,
+                    path='/jobs/deregister/',
+                    data={
+                        'auth_token': config['compute_node_info']['auth_token'],
+                        'public_id': config['compute_node_info']['public_id'],
+                        'error': error,
+                    },
+                )
+            except BaseException as error_object:
+                logger_no_user_data.error(f'Failed to deregister got error: {error_object}')
         else:
             logger_no_user_data.error("Not deregistering as environment is not cloud")
 
@@ -120,34 +116,26 @@ class CloudUtils:
             )
             return
 
-        config = CloudUtils.get_webserver_config()
-
-        for _ in range(100):
-            try:
-                response = requests.post(
-                    url=f'{config["base_url"]}/api/jobs/cloud/finish/',
-                    json={
-                        'auth_token': config["compute_node_info"]["auth_token"],
-                        'cloud_job_id': cloud_job_id,
-                        'system_exception_code': system_exception_code,
-                        'exit_code': exit_code
-                    },
-                    timeout=10,
-                )
-
-                response.raise_for_status()
-
-                opt_error_string = f' with error code {system_exception_code}' if system_exception_code else ''
-                logger_no_user_data.debug(
-                    f'Cloud Job "{cloud_job_id}" was reported as finished' + opt_error_string
-                )
-                return
+        logger_no_user_data.debug(
+            f'Reporting CloudJob "{cloud_job_id}" as finished with exit code: {exit_code} '
+            f'and system exception code: {system_exception_code}'
+        )
 
-            except Exception as error:  # pylint: disable=broad-except
-                logger_no_user_data.debug(
-                    f'Could not finish cloud job "{cloud_job_id}" due to {error}, retrying...'
-                )
-                time.sleep(10)
+        config = CloudUtils.get_webserver_config()
+        try:
+            api.client.post(
+                authenticate=False,
+                path='/jobs/cloud/finish/',
+                retries=100,
+                data={
+                    'auth_token': config['compute_node_info']['auth_token'],
+                    'cloud_job_id': cloud_job_id,
+                    'system_exception_code': system_exception_code,
+                    'exit_code': exit_code
+                },
+            )
+        except BaseException as error:
+            logger_no_user_data.debug(f'Failed to finish CloudJob "{cloud_job_id}" due to: {error}')
 
     @staticmethod
     def _report_availability() -> None:
@@ -158,26 +146,11 @@ class CloudUtils:
             logger_no_user_data.debug(
                 f'Registering with {compute_node_info} to host {api_client.base_url} at {datetime.now()}'
             )
-
-            response: Optional[requests.Response] = None
-            max_retries = 5
-            for retry_count in range(max_retries):
-                try:
-                    response = requests.post(
-                        url=f'{api_client.base_url}/api/jobs/report_available/',
-                        json=compute_node_info,
-                    )
-                    break
-                except Exception as error:  # pylint: disable=broad-except
-                    logger_no_user_data.error(f'Self-registering failed with error: {error}')
-                    if retry_count < max_retries - 1:
-                        seconds_to_sleep = 1
-                        logger_no_user_data.info(f'Retrying self-registering in {seconds_to_sleep} seconds')
-                        time.sleep(seconds_to_sleep)
-
-            if not response:
-                raise BioLibError('Failed to register. Max retry limit reached')
-
+            response = api.client.post(
+                authenticate=False,
+                path='/jobs/report_available/',
+                data=cast(Dict[str, str], compute_node_info),
+            )
             if response.status_code != 201:
                 raise Exception("Non 201 error code")
             else:

biolib/compute_node/job_worker/cache_state.py

@@ -16,7 +16,7 @@ class DockerCacheStateError(CacheStateError):
 
 class LfsCacheState(CacheState):
 
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
 
         self._storage_path_for_write: str = self._get_storage_path_for_write()

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -18,6 +18,8 @@ from docker.errors import ImageNotFound, APIError  # type: ignore
 from docker.models.containers import Container  # type: ignore
 
 from biolib import utils
+
+from biolib._internal.runtime import RuntimeJobDataDict
 from biolib.biolib_binary_format import ModuleInput, ModuleOutputV2
 from biolib.biolib_docker_client import BiolibDockerClient
 from biolib.biolib_errors import DockerContainerNotFoundDuringExecutionException, BioLibError
@@ -37,7 +39,7 @@ from biolib.typing_utils import List, Dict, Optional
 
 class DockerExecutor:
 
-    def __init__(self, options: LocalExecutorOptions):
+    def __init__(self, options: LocalExecutorOptions) -> None:
         self._options: LocalExecutorOptions = options
         self._is_cleaning_up = False
 
@@ -267,12 +269,12 @@ class DockerExecutor:
             internal_network = self._options['internal_network']
             extra_hosts: Dict[str, str] = {}
 
-            biolib_system_secret = {
-                'version': '1.0.0',
-                'job_requested_machine': self._options['job']['requested_machine'],
-                'job_uuid': self._options['job']['public_id'],
-                'job_auth_token': self._options['job']['auth_token'],
-            }
+            biolib_system_secret = RuntimeJobDataDict(
+                version='1.0.0',
+                job_requested_machine=self._options['job']['requested_machine'],
+                job_uuid=self._options['job']['public_id'],
+                job_auth_token=self._options['job']['auth_token'],
+            )
             secrets: Dict[str, str] = dict(
                 **module.get('secrets', {}),
                 biolib_system_secret=json.dumps(biolib_system_secret, indent=4),
@@ -551,7 +553,7 @@ class DockerExecutor:
                 'Docker UpperDir not available. Falling back to container.get_archive() for file extraction'
             )
 
-        logger.debug(f'overlay_upper_dir_path={overlay_upper_dir_path}')
+        logger_no_user_data.debug(f'overlay_upper_dir_path={overlay_upper_dir_path}')
 
         files_and_empty_dirs: List[FileInContainer] = []
         for path in run_diff_paths:

biolib/compute_node/job_worker/job_storage.py

@@ -1,9 +1,8 @@
 import os
 
-import requests
-
 from biolib import utils
-from biolib.biolib_api_client import BiolibApiClient, CreatedJobDict
+from biolib._internal.http_client import HttpClient
+from biolib.biolib_api_client import CreatedJobDict
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.biolib_logging import logger_no_user_data
@@ -15,24 +14,23 @@ class JobStorage:
 
     @staticmethod
     def upload_module_input(job: CreatedJobDict, module_input_serialized: bytes) -> None:
-        base_url = BiolibApiClient.get().base_url
         job_uuid = job['public_id']
         headers = {'Job-Auth-Token': job['auth_token']}
 
         multipart_uploader = utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/start_upload/',
                 headers=headers
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/input/presigned_upload_url/',
                 headers=headers
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/input/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/input/complete_upload/',
                 headers=headers
             ),
         )
@@ -60,7 +58,6 @@ class JobStorage:
 
     @staticmethod
     def _get_module_output_uploader(job_uuid: str) -> utils.MultiPartUploader:
-        base_url = BiolibApiClient.get().base_url
         config = CloudUtils.get_webserver_config()
         compute_node_auth_token = config['compute_node_info']['auth_token']  # pylint: disable=unsubscriptable-object
         headers = {'Compute-Node-Auth-Token': compute_node_auth_token}
@@ -68,17 +65,17 @@ class JobStorage:
         return utils.MultiPartUploader(
             start_multipart_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/start_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/start_upload/',
                 headers=headers,
             ),
             get_presigned_upload_url_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/presigned_upload_url/',
+                path=f'/jobs/{job_uuid}/storage/results/presigned_upload_url/',
                 headers=headers,
             ),
             complete_upload_request=dict(
                 requires_biolib_auth=False,
-                url=f'{base_url}/api/jobs/{job_uuid}/storage/results/complete_upload/',
+                path=f'/jobs/{job_uuid}/storage/results/complete_upload/',
                 headers=headers,
             ),
         )
@@ -92,8 +89,7 @@ class JobStorage:
             job_auth_token=job['auth_token'],
             storage_type='input',
         )
-        response = requests.get(url=presigned_download_url)
-        response.raise_for_status()
+        response = HttpClient.request(url=presigned_download_url)
         data: bytes = response.content
         logger_no_user_data.debug(f'Job "{job_uuid}" module input downloaded')
         return data

biolib/compute_node/job_worker/job_worker.py

@@ -12,9 +12,9 @@ import os
 import signal
 from types import FrameType
 
-import requests
 from docker.models.networks import Network  # type: ignore
 
+from biolib._internal.http_client import HttpClient
 from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
 from biolib.compute_node.job_worker.job_legacy_input_wait_timeout_thread import JobLegacyInputWaitTimeout
 from biolib.compute_node.job_worker.job_storage import JobStorage
@@ -97,7 +97,11 @@ class JobWorker:
         if socket_port:
             self._connect_to_parent()
 
-    def _handle_exit_gracefully(self, signum: int, frame: FrameType) -> None:  # pylint: disable=unused-argument
+    def _handle_exit_gracefully(
+            self,
+            signum: int,
+            frame: Optional[FrameType],  # pylint: disable=unused-argument
+    ) -> None:
         job_id = self._root_job_wrapper["job"]["public_id"] if self._root_job_wrapper else None
         logger_no_user_data.debug(
             f'_JobWorker ({job_id}) got exit signal {signal.Signals(signum).name}'  # pylint: disable=no-member
@@ -246,6 +250,7 @@ class JobWorker:
 
     def _start_network_and_remote_host_proxies(self, job: CreatedJobDict) -> None:
         app_version = job['app_version']
+        app = app_version.get('app', {})
         job_id = job['public_id']
         remote_hosts = app_version['remote_hosts']
         if utils.IS_RUNNING_IN_CLOUD:
@@ -307,7 +312,8 @@ class JobWorker:
                         self._public_network,
                         self._internal_network,
                         job_id,
-                        ports
+                        ports,
+                        can_push_data_record_for_user=app.get('can_push_data_record_for_user', False),
                     )
                     remote_host_proxy.start()
                     self._remote_host_proxies.append(remote_host_proxy)
@@ -526,7 +532,7 @@ class JobWorker:
             start_time = time()
             logger_no_user_data.debug(f'Job "{root_job_id}" downloading runtime zip...')
             try:
-                runtime_zip_bytes = requests.get(runtime_zip_url).content
+                runtime_zip_bytes = HttpClient.request(url=runtime_zip_url).content
             except Exception as exception:
                 raise ComputeProcessException(
                     exception,

biolib/compute_node/remote_host_proxy.py

@@ -14,7 +14,7 @@ from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.typing_utils import Optional, List
 from biolib.biolib_api_client import RemoteHost
 from biolib.biolib_docker_client import BiolibDockerClient
-from biolib.biolib_logging import logger, logger_no_user_data
+from biolib.biolib_logging import logger_no_user_data
 from biolib.biolib_api_client import BiolibApiClient
 
 
@@ -24,8 +24,6 @@ class RemoteHostExtended(RemoteHost):
 
 
 class RemoteHostProxy:
-    _DOCKER_IMAGE_URI = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
-    _TRAFFIC_FORWARDER_PORT_OFFSET = 10000  # Port offset relative to port of a VSOCK proxy
 
     def __init__(
             self,
@@ -33,8 +31,10 @@ class RemoteHostProxy:
             public_network: Network,
             internal_network: Optional[Network],
             job_id: str,
-            ports: List[int]
+            ports: List[int],
+            can_push_data_record_for_user: bool,
     ):
+        self._can_push_data_record_for_user: bool = can_push_data_record_for_user
         self.is_app_caller_proxy = remote_host['hostname'] == 'AppCallerProxy'
 
         # Default to port 443 for now until backend serves remote_hosts with port specified
@@ -52,6 +52,7 @@ class RemoteHostProxy:
         self._job_uuid = job_id
         self._container: Optional[Container] = None
         self._enclave_traffic_forwarder_processes: List[subprocess.Popen] = []
+        self._docker = BiolibDockerClient().get_docker_client()
 
     @property
     def hostname(self) -> str:
@@ -81,7 +82,7 @@ class RemoteHostProxy:
             try:
                 self._container = docker.containers.create(
                     detach=True,
-                    image=self._get_nginx_docker_image(),
+                    image=self._get_biolib_remote_host_proxy_image(),
                     name=self._name,
                     network=self._public_network.name,
                 )
@@ -130,13 +131,23 @@ class RemoteHostProxy:
         for process in self._enclave_traffic_forwarder_processes:
             process.terminate()
 
-    def _get_nginx_docker_image(self) -> Image:
-        docker = BiolibDockerClient.get_docker_client()
+    def _get_biolib_remote_host_proxy_image(self) -> Image:
+        if utils.IS_RUNNING_IN_CLOUD:
+            try:
+                logger_no_user_data.debug('Getting local Docker image for remote host proxy')
+                return self._docker.images.get('biolib-remote-host-proxy:latest')
+            except ImageNotFound:
+                logger_no_user_data.debug(
+                    'Local Docker image for remote host proxy not available. Falling back to public image...'
+                )
+
+        public_image_uri = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
         try:
-            return docker.images.get(self._DOCKER_IMAGE_URI)
+            logger_no_user_data.debug('Getting public Docker image for remote host proxy')
+            return self._docker.images.get(public_image_uri)
         except ImageNotFound:
-            logger.debug('Pulling remote host docker image...')
-            return docker.images.pull(self._DOCKER_IMAGE_URI)
+            logger_no_user_data.debug('Pulling public Docker image for remote host proxy')
+            return self._docker.images.pull(public_image_uri)
 
     def _write_nginx_config_to_container(self, upstream_server_name: str, upstream_server_ports: List[int]) -> None:
         if not self._container:
@@ -156,7 +167,7 @@ class RemoteHostProxy:
 
             if utils.IS_RUNNING_IN_CLOUD:
                 config = CloudUtils.get_webserver_config()
-                s3_results_bucket_name = config['s3_general_storage_bucket_name']  # pylint: disable=unsubscriptable-object
+                s3_results_bucket_name = config['s3_general_storage_bucket_name']
                 s3_results_base_url = f'https://{s3_results_bucket_name}.s3.amazonaws.com'
             else:
                 if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
@@ -196,6 +207,12 @@ http {{
         default    "";
     }}
 
+    map $request_method $bearer_token_on_post_and_get {{
+        POST       "{bearer_token}";
+        GET        "{bearer_token}";
+        default    "";
+    }}
+
     server {{
         listen       80;
         resolver 127.0.0.11 valid=30s;
@@ -270,6 +287,13 @@ http {{
             proxy_ssl_server_name    on;
         }}
 
+        location /api/lfs/ {{
+            proxy_pass               {base_url}/api/lfs/;
+            proxy_set_header         authorization {'$bearer_token_on_post_and_get' if self._can_push_data_record_for_user else '""'};
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
         location /api/ {{
             proxy_pass               {base_url}/api/;
             proxy_set_header         authorization "";
@@ -298,12 +322,25 @@ http {{
             proxy_ssl_server_name    on;
         }}
 
+        {f"""
+        location /proxy/storage/lfs/versions/ {{
+            proxy_pass               {cloud_base_url}/proxy/storage/lfs/versions/;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+        """ if self._can_push_data_record_for_user else ''}
+
         location /proxy/cloud/ {{
             proxy_pass               {cloud_base_url}/proxy/cloud/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
+
+        location / {{
+            return 404 "Not found";
+        }}
     }}
 }}
 '''

biolib/compute_node/webserver/worker_thread.py

@@ -199,7 +199,7 @@ class WorkerThread(threading.Thread):
                 logger_no_user_data.error(f'Job {self._job_uuid} worker process did not exit within 10 seconds')
                 if utils.IS_RUNNING_IN_CLOUD:
                     logger_no_user_data.error('Deregistering compute node...')
-                    CloudUtils.deregister()
+                    CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
         # Delete result as error occurred
@@ -249,7 +249,7 @@ class WorkerThread(threading.Thread):
 
         if utils.IS_RUNNING_IN_CLOUD:
             if deregistered_due_to_error:
-                CloudUtils.shutdown() # shutdown now
+                CloudUtils.shutdown()  # shutdown now
             else:
                 webserver_utils.update_auto_shutdown_time()
 

biolib/jobs/job.py

@@ -6,9 +6,10 @@ from pathlib import Path
 from collections import OrderedDict
 from urllib.parse import urlparse
 
-import requests
 from biolib import api, utils
 from biolib._internal.http_client import HttpClient
+from biolib._internal.utils import open_browser_window_from_notebook
+from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_api_client.biolib_job_api import BiolibJobApi
 from biolib.biolib_binary_format import LazyLoadedFile, ModuleOutputV2, ModuleInput, ModuleInputDict
 from biolib.biolib_binary_format.stdout_and_stderr import StdoutAndStderr
@@ -24,12 +25,14 @@ from biolib.utils import IS_RUNNING_IN_NOTEBOOK
 
 class Job:
     # Columns to print in table when showing Job
-    table_columns_to_row_map = OrderedDict({
-        'ID': {'key': 'uuid', 'params': {'width': 36}},
-        'Application': {'key': 'app_uri', 'params': {}},
-        'Status': {'key': 'state', 'params': {}},
-        'Started At': {'key': 'started_at', 'params': {}},
-    })
+    table_columns_to_row_map = OrderedDict(
+        {
+            'ID': {'key': 'uuid', 'params': {'width': 36}},
+            'Application': {'key': 'app_uri', 'params': {}},
+            'Status': {'key': 'state', 'params': {}},
+            'Started At': {'key': 'started_at', 'params': {}},
+        }
+    )
 
     def __init__(self, job_dict: JobDict):
         self._uuid: str = job_dict['uuid']
@@ -127,8 +130,7 @@ class Job:
             job_auth_token=self._job_dict['auth_token'],
             storage_type='input',
         )
-        response = requests.get(url=presigned_download_url)
-        response.raise_for_status()
+        response = HttpClient.request(url=presigned_download_url)
         module_input_serialized: bytes = response.content
         return ModuleInput(module_input_serialized).deserialize()
 
@@ -174,6 +176,17 @@ class Job:
             time.sleep(2)
         logger.info(f'Job {self.id} has finished.')
 
+    def open_browser(self) -> None:
+        api_client = BiolibApiClient.get()
+        results_url_to_open = f'{api_client.base_url}/results/{self.id}/?token={self._auth_token}'
+        if IS_RUNNING_IN_NOTEBOOK:
+            print(f'Opening results page at: {results_url_to_open}')
+            print('If your browser does not open automatically, click on the link above.')
+            open_browser_window_from_notebook(results_url_to_open)
+        else:
+            print('Please copy and paste the following link into your browser:')
+            print(results_url_to_open)
+
     def _get_cloud_job(self) -> CloudJobDict:
         self._refetch_job_dict(force_refetch=True)
         if self._job_dict['cloud_job'] is None:
@@ -192,20 +205,11 @@ class Job:
     @staticmethod
     def show_jobs(count: int = 25) -> None:
         job_dicts = Job._get_job_dicts(count)
-        BioLibTable(
-            columns_to_row_map=Job.table_columns_to_row_map,
-            rows=job_dicts,
-            title='Jobs'
-        ).print_table()
+        BioLibTable(columns_to_row_map=Job.table_columns_to_row_map, rows=job_dicts, title='Jobs').print_table()
 
     @staticmethod
     def _get_job_dicts(count: int) -> List['JobDict']:
-        job_dicts: List['JobDict'] = api.client.get(
-            path='/jobs/',
-            params={
-                'page_size': str(count)
-            }
-        ).json()['results']
+        job_dicts: List['JobDict'] = api.client.get(path='/jobs/', params={'page_size': str(count)}).json()['results']
         return job_dicts
 
     @staticmethod
@@ -237,9 +241,7 @@ class Job:
     def show(self) -> None:
         self._refetch_job_dict()
         BioLibTable(
-            columns_to_row_map=Job.table_columns_to_row_map,
-            rows=[self._job_dict],
-            title=f'Job: {self._uuid}'
+            columns_to_row_map=Job.table_columns_to_row_map, rows=[self._job_dict], title=f'Job: {self._uuid}'
         ).print_table()
 
     def stream_logs(self) -> None:
@@ -274,6 +276,10 @@ class Job:
         while True:
             time.sleep(2)
             status_json = self._get_job_status_from_compute_node(compute_node_url)
+            if not status_json:
+                # this can happen if the job is finished but already removed from the compute node
+                logger.warning("WARN: We were unable to retrieve the full log of the job, please try again")
+                break
             job_is_completed = status_json['is_completed']
             for status_update in status_json['status_updates']:
                 # If the job is completed, print the log messages after all stdout and stderr has been written
@@ -303,12 +309,10 @@ class Job:
 
     def _print_full_logs(self, node_url: str) -> None:
         try:
-            response_json = HttpClient.request(
-                url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full'
-            ).json()
+            response_json = HttpClient.request(url=f'{node_url}/v1/job/{self._uuid}/status/?logs=full').json()
         except Exception as error:
             logger.error(f'Could not get full streamed logs due to: {error}')
-            raise BioLibError from error
+            raise BioLibError('Could not get full streamed logs') from error
 
         for status_update in response_json.get('previous_status_updates', []):
             logger.info(f'Cloud: {status_update["log_message"]}')
@@ -334,9 +338,7 @@ class Job:
     def _get_job_status_from_compute_node(self, compute_node_url):
         for _ in range(15):
             try:
-                return HttpClient.request(
-                    url=f'{compute_node_url}/v1/job/{self._uuid}/status/'
-                ).json()
+                return HttpClient.request(url=f'{compute_node_url}/v1/job/{self._uuid}/status/').json()
             except Exception:  # pylint: disable=broad-except
                 cloud_job = self._get_cloud_job()
                 logger.debug("Failed to get status from compute node, retrying...")
@@ -344,8 +346,7 @@ class Job:
                     logger.debug("Job no longer exists on compute node, checking for error...")
                     if cloud_job['error_code'] != SystemExceptionCodes.COMPLETED_SUCCESSFULLY.value:
                         error_message = SystemExceptionCodeMap.get(
-                            cloud_job['error_code'],
-                            f'Unknown error code {cloud_job["error_code"]}'
+                            cloud_job['error_code'], f'Unknown error code {cloud_job["error_code"]}'
                         )
                         raise BioLibError(f'Cloud: {error_message}') from None
                     else:

biolib/lfs/__init__.py

@@ -1,6 +1,4 @@
 from .cache import prune_lfs_cache
 from .utils import \
-    describe_large_file_system, \
-    get_file_data_from_large_file_system, \
     push_large_file_system, \
     create_large_file_system