pybiolib 1.1.2193__py3-none-any.whl → 1.2.7.dev0__py3-none-any.whl

biolib/_internal/data_record/data_record.py

@@ -83,6 +83,8 @@ def verify_schema(specification: SqliteV1DatabaseSchema, actual_schema: SqliteV1
 
 def get_data_record_state_from_uri(uri) -> 'DataRecordVersionInfo':
     app_response: AppGetResponse = api_client.get(path='/app/', params={'uri': uri}).json()
+    if app_response['app']['type'] != 'data-record':
+        raise Exception(f'Resource "{uri}" is not a Data Record')
     return DataRecordVersionInfo(
         resource_uri=app_response['app_version']['app_uri'],
         resource_uuid=app_response['app']['public_id'],

biolib/_internal/http_client.py

@@ -125,12 +125,14 @@ class HttpClient:
 
             except urllib.error.URLError as error:
                 if isinstance(error.reason, socket.timeout):
-                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                    if retry_count > 0:
+                        logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                     last_error = error
                 else:
                     raise error
             except socket.timeout as error:
-                logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
+                if retry_count > 0:
+                    logger_no_user_data.warning(f'HTTP {method} request failed with read timeout for "{url}"')
                 last_error = error
 
         raise last_error or Exception(f'HTTP {method} request failed after {retries} retries for "{url}"')

biolib/_internal/push_application.py

@@ -211,6 +211,11 @@ def push_application(
             try:
                 logger.info(f'Trying to push image {docker_image_name} defined on module {module_name}.')
                 image = docker_client.images.get(docker_image_name)
+                architecture = image.attrs.get('Architecture')
+                if architecture != 'amd64':
+                    print(f"Error: '{docker_image_name}' is compiled for {architecture}, expected x86 (amd64).")
+                    print('If you are on an ARM processor, try passing --platform linux/amd64 to docker build.')
+                    exit(1)
                 absolute_repo_uri = f'{utils.BIOLIB_SITE_HOSTNAME}/{repo}'
                 image.tag(absolute_repo_uri, tag)
 

biolib/_internal/runtime.py

@@ -7,6 +7,7 @@ class RuntimeJobDataDict(TypedDict):
     job_uuid: str
     job_auth_token: str
     app_uri: str
+    is_environment_biolib_cloud: bool
 
 
 class BioLibRuntimeError(Exception):

biolib/_internal/utils/multinode.py

@@ -0,0 +1,264 @@
+import glob
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+
+import biolib
+from biolib.utils import SeqUtil
+
+
+def natsorted(lst):
+    """Sort the list using the natural sort key."""
+
+    def _natural_sort_key(s):
+        """A key function for natural sorting."""
+        return [int(text) if text.isdigit() else text.lower() for text in re.split('([0-9]+)', s)]
+
+    return sorted(lst, key=_natural_sort_key)
+
+
+def fasta_above_threshold(fasta_file, work_threshold, work_per_residue=1, verbose=False):
+    """True if total FASYA residue work above max_work"""
+
+    records = SeqUtil.parse_fasta(fasta_file)
+
+    # Calculate work units
+    total_work_units = 0
+    for i, record in enumerate(records):
+        sequence_work_units = len(record.sequence) * work_per_residue
+        total_work_units += sequence_work_units
+
+        if total_work_units >= work_threshold:
+            if verbose:
+                print(f'FASTA above threshold (stopped at {total_work_units}) >= {work_threshold}')
+                print(f'From  from {i+1}/{len(records)} sequences in {fasta_file}')
+            return True
+
+    if verbose:
+        print(f'FASTA below threshold ({total_work_units}) < {work_threshold}')
+        print(f'From {len(records)} sequences in {fasta_file}')
+
+    return False
+
+
+def run_locally(command_list, args):
+    """Run script locally (no multi-node processing)"""
+
+    # Prepare command
+    new_args = vars(args)
+
+    # Delete multinode-specific input arguments
+    for k in list(new_args.keys()):
+        if str(k).startswith('multinode'):
+            del new_args[k]
+
+    # Convert to list format
+    new_args_list = _args_dict_to_args_list(new_args)
+
+    # Prepare command, e.g. ["python3", "predict.py"] + new_args_list
+    command = command_list + new_args_list
+
+    if args.verbose >= 1:
+        print(f'Running {command}')
+
+    # Run command
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode == 0:
+        print(f'{result.stdout}')
+    else:
+        print(f'Error: {result.stderr}')
+
+
+def fasta_batch_records(fasta_file, work_per_batch_min, work_per_residue=1, verbose=False):
+    """Converts FASTA records to batches of records, based on thresholds"""
+
+    def log_batches(batches):
+        for i, batch in enumerate(batches):
+            batch_dict = {
+                'records': len(batch),
+                'residues': sum(len(record.sequence) for record in batch),
+            }
+
+            n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+            print(f'Batch {i+1}: {n_res} residues from {n_seqs} sequences')
+
+    records = SeqUtil.parse_fasta(fasta_file)
+
+    batches = []
+    batch = []
+    current_work_units = 0
+    total_work_units = 0
+    for record in records:
+        # Add to batch
+        batch.append(record)
+
+        # Calculate work units
+        seq = record.sequence
+        sequence_work_units = len(seq) * work_per_residue
+
+        # Increase counters
+        current_work_units += sequence_work_units
+        total_work_units += sequence_work_units
+
+        # If above limit, start a new batch
+        if current_work_units >= work_per_batch_min:
+            batches.append(batch)
+            batch = []
+            current_work_units = 0
+
+    # Append last batch if present
+    if batch:
+        batches.append(batch)
+
+    if verbose:
+        log_batches(batches)
+
+    return batches
+
+
+def fasta_send_batches_biolib(app_url, batches, args, args_fasta='fasta', verbose=1):
+    """
+    Send jobs through pybiolib interface
+    """
+
+    if args.verbose >= 1:
+        print(f'Sending {len(batches)} batches to Biolib')
+
+    # Login to biolib, prepare app
+    # current_app = biolib.load(Runtime.get_app_uri())
+    biolib.login()
+    current_app = biolib.load(app_url)  # Nb: uses "_" not "-"
+
+    # Compute results
+    job_list = []
+    for i, batch_records in enumerate(batches):  # MH
+        # Write FASTA, send to server
+        with tempfile.TemporaryDirectory() as tempdir:
+            # New arguments
+            new_args = vars(args)
+
+            # Write batched FASTA to send
+            fasta_path = f'{tempdir}/input.fasta'
+            SeqUtil.write_records_to_fasta(fasta_path, batch_records)
+            new_args[args_fasta] = fasta_path
+            new_args['multinode_only_local'] = True
+
+            # Convert to list
+            new_args_list = _args_dict_to_args_list(new_args)
+
+            # Send job
+            job = current_app.cli(args=new_args_list, blocking=False)
+            job_list.append(job)
+
+            # Job stats
+            if args.verbose:
+                batch_dict = _get_batch_stats(batch_records)
+                n_seqs, n_res = batch_dict['records'], batch_dict['residues']
+                print(f'Sending job {i+1}: {n_res} residues from {n_seqs} sequences -> arg_list = {new_args_list}')
+
+    # Stream job output at a time
+    print('Streaming job outputs ...')
+    for i, job in enumerate(job_list):
+        job.stream_logs()
+
+        # Check if job succeeded
+        assert job.get_exit_code() == 0, f'Job failed with exit code {job.get_exit_code()}'
+
+        # Write to disk
+        output_dir = f'job_output/job_{i+1}'
+        job.save_files(output_dir=output_dir)
+
+        if verbose:
+            print(f'Saving to {output_dir}')
+
+
+def merge_folder(folder_name, job_out_dir='job_output', out_dir='output', verbose=1):
+    """Helper function for merging folders"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_folder = f'{job_dirs[0]}/{folder_name}'
+    merged_folder = f'{out_dir}/{folder_name}'
+    shutil.move(first_folder, merged_folder)
+
+    if verbose:
+        print(f'Merging {folder_name} from {len(job_dirs)} directories to {merged_folder}')
+
+    # If more than one folder, merge to first
+    if len(job_dirs) >= 2:
+        # Find each job output file
+        for job_dir in job_dirs[1:]:
+            # Move over extra files
+            extra_folder = f'{job_dir}/{folder_name}'
+            extra_files = os.listdir(extra_folder)
+            for file_name in extra_files:
+                file_path = f'{extra_folder}/{file_name}'
+                shutil.move(file_path, merged_folder)
+
+
+def merge_file(
+    file_name,
+    header_lines_int=1,
+    job_out_dir='job_output',
+    out_dir='output',
+    verbose=1,
+):
+    """Helper function for merging files with headers"""
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    job_dirs = glob.glob(f'{job_out_dir}/job_*')
+    job_dirs = natsorted(job_dirs)
+
+    # Move first file, prepare to merge
+    first_file = f'{job_dirs[0]}/{file_name}'
+    merged_file = f'{out_dir}/{file_name}'
+    shutil.move(first_file, merged_file)
+
+    if verbose:
+        print(f'Merging {file_name} from {len(job_dirs)} directories to {merged_file}')
+
+    # If more than one file, append to first
+    if len(job_dirs) >= 2:
+        # Open first file
+        with open(merged_file, 'a') as merged_file_handle:
+            # Find each job output file
+            for job_dir in job_dirs[1:]:
+                # Open extra file
+                extra_file = f'{job_dir}/{file_name}'
+                with open(extra_file) as extra_file_handle:
+                    # Skip first n header lines
+                    for _ in range(header_lines_int):
+                        next(extra_file_handle)
+
+                    # Append content to first file
+                    contents = extra_file_handle.read()
+                    merged_file_handle.write(contents)
+
+
+def _get_batch_stats(batch):
+    stats_dict = {
+        'records': len(batch),
+        'residues': sum(len(R.sequence) for R in batch),
+    }
+
+    return stats_dict
+
+
+def _args_dict_to_args_list(new_args):
+    """Converts args dict to list of arguments for Biolib"""
+
+    nested_list = [[f'--{key}', f'{value}'] for key, value in new_args.items()]
+
+    arg_list = []
+    for lst in nested_list:
+        for item in lst:
+            arg_list.append(item)
+
+    return arg_list

biolib/_runtime/runtime.py

@@ -14,6 +14,10 @@ class Runtime:
     def check_is_environment_biolib_app() -> bool:
         return bool(Runtime._try_to_get_job_data())
 
+    @staticmethod
+    def check_is_environment_biolib_cloud() -> bool:
+        return Runtime._get_job_data().get('is_environment_biolib_cloud', False)
+
     @staticmethod
     def get_job_id() -> str:
         return Runtime._get_job_data()['job_uuid']

biolib/biolib_api_client/app_types.py

@@ -32,6 +32,7 @@ class App(TypedDict):
     public_id: str
     state: str
     resource_uri: str
+    type: str
 
 
 class AppGetResponse(TypedDict):

biolib/cli/data_record.py

@@ -6,6 +6,7 @@ from typing import Dict, List
 import click
 
 from biolib._data_record.data_record import DataRecord
+from biolib.biolib_api_client import BiolibApiClient
 from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.typing_utils import Optional
 
@@ -57,6 +58,7 @@ def download(uri: str, file: Optional[str], path_filter: Optional[str]) -> None:
 @click.argument('uri', required=True)
 @click.option('--json', 'output_as_json', is_flag=True, default=False, required=False, help='Format output as JSON')
 def describe(uri: str, output_as_json: bool) -> None:
+    BiolibApiClient.assert_is_signed_in(authenticated_action_description='get Data Record description')
     record = DataRecord.get_by_uri(uri)
     files_info: List[Dict] = []
     total_size_in_bytes = 0

biolib/compute_node/job_worker/executors/docker_executor.py

@@ -286,6 +286,7 @@ class DockerExecutor:
                 job_uuid=self._options['job']['public_id'],
                 job_auth_token=self._options['job']['auth_token'],
                 app_uri=self._options['job']['app_uri'],
+                is_environment_biolib_cloud=bool(utils.IS_RUNNING_IN_CLOUD),
             )
             secrets: Dict[str, str] = dict(
                 **module.get('secrets', {}),

biolib/compute_node/job_worker/job_storage.py

@@ -47,8 +47,21 @@ class JobStorage:
         module_output_path = os.path.join(job_temporary_dir, JobStorage.module_output_file_name)
         module_output_size = os.path.getsize(module_output_path)
 
+        # Calculate chunk size based on max chunk count of 10_000, using 9_000 to be on the safe side
+        max_chunk_count = 9_000
+        min_chunk_size_bytes = 50_000_000
+        chunk_size_in_bytes = max(min_chunk_size_bytes, module_output_size // max_chunk_count)
+
+        logger_no_user_data.debug(
+            f'Job "{job_uuid}" uploading result of size {module_output_size} bytes '
+            f'with chunk size of {chunk_size_in_bytes} bytes...'
+        )
+
         with open(module_output_path, mode='rb') as module_output_file:
-            module_output_iterator = get_chunk_iterator_from_file_object(module_output_file)
+            module_output_iterator = get_chunk_iterator_from_file_object(
+                file_object=module_output_file,
+                chunk_size_in_bytes=chunk_size_in_bytes,
+            )
             multipart_uploader = JobStorage._get_module_output_uploader(job_uuid)
             multipart_uploader.upload(
                 payload_iterator=module_output_iterator,

biolib/compute_node/remote_host_proxy.py

@@ -1,3 +1,4 @@
+import base64
 import io
 import subprocess
 import tarfile
@@ -160,6 +161,9 @@ class RemoteHostProxy:
             access_token = BiolibApiClient.get().access_token
             bearer_token = f'Bearer {access_token}' if access_token else ''
 
+            biolib_index_basic_auth = f'compute_node|admin:{compute_node_auth_token},{self._job_uuid}'
+            biolib_index_basic_auth_base64 = base64.b64encode(biolib_index_basic_auth.encode('utf-8')).decode('utf-8')
+
             nginx_config = f"""
 events {{
   worker_connections  1024;
@@ -270,7 +274,7 @@ http {{
         }}
 
         location /api/lfs/ {{
-            proxy_pass               https://$upstream_hostname/api/lfs/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
             proxy_set_header         job-uuid "{self._job_uuid}";
@@ -279,7 +283,7 @@ http {{
         }}
 
         location /api/app/ {{
-            proxy_pass               https://$upstream_hostname/api/app/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
             proxy_set_header         job-uuid "{self._job_uuid}";
@@ -288,33 +292,40 @@ http {{
         }}
 
         location /api/ {{
-            proxy_pass               https://$upstream_hostname/api/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/job-storage/ {{
-            proxy_pass               https://$upstream_hostname/proxy/storage/job-storage/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/lfs/versions/ {{
-            proxy_pass               https://$upstream_hostname/proxy/storage/lfs/versions/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/cloud/ {{
-            proxy_pass               https://$upstream_hostname/proxy/cloud/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
+        location /proxy/index/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "Basic {biolib_index_basic_auth_base64}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
         location / {{
             return 404 "Not found";
         }}

biolib/compute_node/webserver/worker_thread.py

@@ -2,23 +2,23 @@ import base64
 import os
 import random
 import shutil
+import socket
 import sys
-import time
 import threading
-import socket
+import time
 from queue import Queue
 
-from biolib import utils
+from biolib import api, utils
+from biolib.biolib_binary_format import ModuleOutputV2, SystemException, SystemStatusUpdate
 from biolib.biolib_binary_format.utils import LocalFileIndexableBuffer
+from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker import JobWorkerProcess
 from biolib.compute_node.job_worker.job_storage import JobStorage
 from biolib.compute_node.socker_listener_thread import SocketListenerThread
 from biolib.compute_node.socket_sender_thread import SocketSenderThread
+from biolib.compute_node.utils import SystemExceptionCodes, WorkerThreadException, get_package_type
 from biolib.compute_node.webserver import webserver_utils
-from biolib.biolib_binary_format import SystemStatusUpdate, SystemException, ModuleOutputV2
-from biolib.compute_node.utils import get_package_type, WorkerThreadException, SystemExceptionCodes
-from biolib.biolib_logging import logger, logger_no_user_data
 
 SOCKET_HOST = '127.0.0.1'
 
@@ -37,7 +37,7 @@ class WorkerThread(threading.Thread):
             self._sender_thread = None
             self._start_and_connect_to_compute_process()
 
-            logger.debug(f"WorkerThread connected to port {self._socket_port}")
+            logger.debug(f'WorkerThread connected to port {self._socket_port}')
 
         except Exception as exception:
             logger_no_user_data.error(exception)
@@ -79,20 +79,16 @@ class WorkerThread(threading.Thread):
                     if progress == 94:
                         # Get Job exit code
                         try:
-                            module_output_path = os.path.join(self._job_temporary_dir,
-                                                              JobStorage.module_output_file_name)
-                            module_output = ModuleOutputV2(
-                                buffer=LocalFileIndexableBuffer(
-                                    filename=module_output_path
-                                )
+                            module_output_path = os.path.join(
+                                self._job_temporary_dir,
+                                JobStorage.module_output_file_name,
                             )
+                            module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
                             self.compute_state['exit_code'] = module_output.get_exit_code()
                             logger_no_user_data.debug(f"Got exit code: {self.compute_state['exit_code']}")
 
                         except Exception as error:  # pylint: disable=broad-except
-                            logger_no_user_data.error(
-                                f'Could not get exit_code from module output due to: {error}'
-                            )
+                            logger_no_user_data.error(f'Could not get exit_code from module output due to: {error}')
 
                         if utils.IS_RUNNING_IN_CLOUD:
                             JobStorage.upload_module_output(
@@ -107,7 +103,7 @@ class WorkerThread(threading.Thread):
                 elif package_type == 'SystemException':
                     error_code = SystemException(package).deserialize()
                     self.compute_state['status']['error_code'] = error_code
-                    logger.debug("Hit error. Terminating Worker Thread and Compute Process")
+                    logger.debug('Hit error. Terminating Worker Thread and Compute Process')
                     self.compute_state['progress'] = 95
                     self.terminate()
 
@@ -153,10 +149,10 @@ class WorkerThread(threading.Thread):
 
         # Starting a thread for accepting connections before starting the process that should to connect to the socket
         logger_no_user_data.debug('Starting connection thread')
-        self._connection_thread = threading.Thread(target=self._accept_new_socket_connection, args=[
-            received_messages_queue,
-            messages_to_send_queue
-        ])
+        self._connection_thread = threading.Thread(
+            target=self._accept_new_socket_connection,
+            args=[received_messages_queue, messages_to_send_queue],
+        )
         self._connection_thread.start()
         logger_no_user_data.debug('Started connection thread')
         logger_no_user_data.debug('Starting compute process')
@@ -177,6 +173,16 @@ class WorkerThread(threading.Thread):
         self._sender_thread.start()
 
     def terminate(self) -> None:
+        cloud_job_uuid = self.compute_state['cloud_job_id']
+        exit_code = self.compute_state.get('exit_code')
+        system_exception_code = self.compute_state['status'].get('error_code')
+        if utils.IS_RUNNING_IN_CLOUD:
+            CloudUtils.finish_cloud_job(
+                cloud_job_id=cloud_job_uuid,
+                system_exception_code=system_exception_code,
+                exit_code=exit_code,
+            )
+
         deregistered_due_to_error = False
         if self._job_worker_process:
             logger_no_user_data.debug(
@@ -184,7 +190,8 @@ class WorkerThread(threading.Thread):
             )
             self._job_worker_process.terminate()
 
-            for _ in range(10):
+            clean_up_timeout_in_seconds = 600
+            for _ in range(clean_up_timeout_in_seconds):
                 if self._job_worker_process.exitcode is not None:
                     logger_no_user_data.debug(
                         f'Job "{self._job_uuid}" worker process exitcode {self._job_worker_process.exitcode}'
@@ -196,28 +203,18 @@ class WorkerThread(threading.Thread):
 
             if self._job_worker_process.exitcode is None:
                 # TODO: Figure out if more error handling is necessary here
-                logger_no_user_data.error(f'Job {self._job_uuid} worker process did not exit within 10 seconds')
+                logger_no_user_data.error(
+                    f'Job {self._job_uuid} worker process did not exit within {clean_up_timeout_in_seconds} seconds'
+                )
                 if utils.IS_RUNNING_IN_CLOUD:
                     logger_no_user_data.error('Deregistering compute node...')
                     CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
         # Delete result as error occurred
-        system_exception_code = self.compute_state['status'].get('error_code')
         if system_exception_code and os.path.exists(self._job_temporary_dir):
             shutil.rmtree(self._job_temporary_dir)
 
-        exit_code = self.compute_state.get('exit_code')
-
-        if utils.IS_RUNNING_IN_CLOUD:
-            # Get and send compute node exception code and job exit code if present
-            logger_no_user_data.debug(f"Sending exit code {exit_code}")
-            CloudUtils.finish_cloud_job(
-                cloud_job_id=self.compute_state['cloud_job_id'],
-                system_exception_code=system_exception_code,
-                exit_code=exit_code
-            )
-
         if self._socket:
             self._socket.close()
 
@@ -225,7 +222,7 @@ class WorkerThread(threading.Thread):
             self._connection.close()
 
         if self.compute_state['progress'] == 95:
-            seconds_to_sleep = 60  # 1 minute
+            seconds_to_sleep = 5
             logger_no_user_data.debug(
                 f'Job "{self._job_uuid}" worker thread sleeping for {seconds_to_sleep} seconds before cleaning up'
             )
@@ -234,7 +231,7 @@ class WorkerThread(threading.Thread):
 
         compute_state_dict = webserver_utils.JOB_ID_TO_COMPUTE_STATE_DICT
         if self._job_uuid in compute_state_dict:
-            # Delete result as user has not started download within 60 seconds
+            # Delete result as user has not started download
             if compute_state_dict[self._job_uuid]['progress'] == 95 and os.path.exists(self._job_temporary_dir):
                 shutil.rmtree(self._job_temporary_dir)
 
@@ -245,12 +242,18 @@ class WorkerThread(threading.Thread):
                 f'Job "{self._job_uuid}" could not be found, maybe it has already been cleaned up'
             )
 
-        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread terminated')
-
         if utils.IS_RUNNING_IN_CLOUD:
+            config = CloudUtils.get_webserver_config()
+            logger_no_user_data.debug(f'Job "{self._job_uuid}" reporting CloudJob "{cloud_job_uuid}" as cleaned up...')
+            api.client.post(
+                path=f'/internal/compute-nodes/cloud-jobs/{cloud_job_uuid}/cleaned-up/',
+                headers={'Compute-Node-Auth-Token': config['compute_node_info']['auth_token']},
+            )
+
             if deregistered_due_to_error:
                 CloudUtils.shutdown()  # shutdown now
             else:
                 webserver_utils.update_auto_shutdown_time()
 
+        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread exiting...')
         sys.exit()

{pybiolib-1.1.2193.dist-info → pybiolib-1.2.7.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybiolib
-Version: 1.1.2193
+Version: 1.2.7.dev0
 Summary: BioLib Python Client
 Home-page: https://github.com/biolib
 License: MIT

{pybiolib-1.1.2193.dist-info → pybiolib-1.2.7.dev0.dist-info}/RECORD

@@ -4,18 +4,18 @@ biolib/__init__.py,sha256=_tThyzISH81yS9KXP_X3qEiKXmsIp5XOBcJIODfLVnc,4338
 biolib/_data_record/data_record.py,sha256=CoyYRse5VdUBhQzzPfR9BkytgOsM-IZxkfMX1kyRnPk,12589
 biolib/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/_internal/data_record/__init__.py,sha256=fGdME6JGRU_2VxpJbYpGXYndjN-feUkmKY4fuMyq3cg,76
-biolib/_internal/data_record/data_record.py,sha256=YmaAABR57goDCE8-rKb2j0FPMSbDtRPCm_HhT3mM074,4299
+biolib/_internal/data_record/data_record.py,sha256=g_-jdy5-Zem3dthwxJj2OuQqkDGTyc-iGqN1rtYYD1A,4418
 biolib/_internal/data_record/remote_storage_endpoint.py,sha256=eCptuZ4DMAPnaNCVDvpWXwXGI6Jac9U1N5dqU8Cj95Q,1732
 biolib/_internal/file_utils.py,sha256=4jT6j7bB21c0JNn5BfnyWQib_zt0CVtJ_TiOFOStRcE,2604
 biolib/_internal/fuse_mount/__init__.py,sha256=B_tM6RM2dBw-vbpoHJC4X3tOAaN1H2RDvqYJOw3xFwg,55
 biolib/_internal/fuse_mount/experiment_fuse_mount.py,sha256=08aUdEq_bvqLBft_gSLjOClKDy5sBnMts1RfJf7AP_U,7012
-biolib/_internal/http_client.py,sha256=DdooXei93JKGYGV4aQmzue_oFzvHkozg2UCxgk9dfDM,5081
+biolib/_internal/http_client.py,sha256=Q7z7_DMI1EaVJSyevrra63piuee-NfqcAHRtQt76ZdA,5165
 biolib/_internal/lfs/__init__.py,sha256=gSWo_xg61UniYgD7yNYxeT4I9uaXBCBSi3_nmZjnPpE,35
 biolib/_internal/lfs/cache.py,sha256=pQS2np21rdJ6I3DpoOutnzPHpLOZgUIS8TMltUJk_k4,2226
 biolib/_internal/libs/__init__.py,sha256=Jdf4tNPqe_oIIf6zYml6TiqhL_02Vyqwge6IELrAFhw,98
 biolib/_internal/libs/fusepy/__init__.py,sha256=AWDzNFS-XV_5yKb0Qx7kggIhPzq1nj_BZS5y2Nso08k,41944
-biolib/_internal/push_application.py,sha256=8P7eXvySn7CRp5XBDkO3xjTGixS8g7-jD-_iwzM_XDI,10020
-biolib/_internal/runtime.py,sha256=9pZ3s3L7LGxdqOgnHh1KK3Jjyn_9MjhQmKHI-6hMT3U,448
+biolib/_internal/push_application.py,sha256=b1WvlijJYOLp7f43AoAzjINKkALM9DVB8yfSetEP4Z4,10381
+biolib/_internal/runtime.py,sha256=BiHl4klUHr36MCpqKaUso4idHeBZfPAahLYRQrabFqA,486
 biolib/_internal/types/__init__.py,sha256=11ZucS8jKeLGAAswXyKI7FH2KLHd6T9Sh8ZK2Ar3jlk,152
 biolib/_internal/types/app.py,sha256=Mz2QGD_jESX-K9JYnLWPo4YA__Q_1FQQTk9pvidCohU,118
 biolib/_internal/types/data_record.py,sha256=AHoIiwVqeHj0HozQxFRAyxk-d3XJgLWno4ic1z9eTrQ,865
@@ -23,7 +23,8 @@ biolib/_internal/types/experiment.py,sha256=D94iBdn2nS92lRW-TOs1a2WKXJD5ZtmzL4yp
 biolib/_internal/types/resource.py,sha256=G-vPkZoe4Um6FPxsQZtRzAlbSW5sDW4NFkbjn21I3V4,372
 biolib/_internal/types/typing.py,sha256=D4EKKEe7kDx0K6lJi-H_XLtk-8w6nu2fdqn9bvzI-Xo,288
 biolib/_internal/utils/__init__.py,sha256=p5vsIFyu-zYqBgdSMfwW9NC_jk7rXvvCbV4Bzd3As7c,630
-biolib/_runtime/runtime.py,sha256=daYxzIpRoW4k-HJFu2BMXeylYSlCXn3-SqdSriCFnKw,2770
+biolib/_internal/utils/multinode.py,sha256=UnM08GXc8U-p0eoSleer4BIgngIsn_fgh9FxRQJkIiI,8068
+biolib/_runtime/runtime.py,sha256=oVgTnDDJv9L4BUP1_sd0oAj4LLyyiPSQdhp7ixWARvw,2923
 biolib/api/__init__.py,sha256=mQ4u8FijqyLzjYMezMUUbbBGNB3iFmkNdjXnWPZ7Jlw,138
 biolib/api/client.py,sha256=FRpdH5aI187b_I_4HUNi680v4iOP65z5f2RcUo8D8MA,3559
 biolib/app/__init__.py,sha256=cdPtcfb_U-bxb9iSL4fCEq2rpD9OjkyY4W-Zw60B0LI,37
@@ -31,7 +32,7 @@ biolib/app/app.py,sha256=P2RwaDAskUHzlciuTJUroqUocRwoyOLT6YbgMyCRRDI,8484
 biolib/app/search_apps.py,sha256=K4a41f5XIWth2BWI7OffASgIsD0ko8elCax8YL2igaY,1470
 biolib/biolib_api_client/__init__.py,sha256=E5EMa19wJoblwSdQPYrxc_BtIeRsAuO0L_jQweWw-Yk,182
 biolib/biolib_api_client/api_client.py,sha256=ciNx4ybpyKG5LEf4KQdGEz13r0jTxImyQat4_HDecD0,7373
-biolib/biolib_api_client/app_types.py,sha256=FxSr4UqfnMhLe34p8bm02wsC3g1Jz8iaing5tRKDOQI,2442
+biolib/biolib_api_client/app_types.py,sha256=1sXz9XnLRKNALMglNdTbew7AL6OkcUan0MPdj4xQLis,2456
 biolib/biolib_api_client/auth.py,sha256=kjm0ZHnH3I8so3su2sZbBxNHYp-ZUdrZ5lwQ0K36RSw,949
 biolib/biolib_api_client/biolib_app_api.py,sha256=DndlVxrNTes6DOaWyMINLGZQCRMWVvR7gwt5HVlyf5Y,4240
 biolib/biolib_api_client/biolib_job_api.py,sha256=7bKfav3-12ewXkEUoLdCmbWdebW8148kxfGJW9SsXZI,7125
@@ -57,7 +58,7 @@ biolib/biolib_errors.py,sha256=5m4lK2l39DafpoXBImEBD4EPH3ayXBX0JgtPzmGClow,689
 biolib/biolib_logging.py,sha256=J3E5H_LL5k6ZUim2C8gqN7E6lCBZMTpO4tnMpOPwG9U,2854
 biolib/cli/__init__.py,sha256=0v3c_J-U0k46c5ZWeQjLG_kTaKDJm81LBxQpDO2B_aI,1286
 biolib/cli/auth.py,sha256=rpWGmXs6Fz6CGrO9K8ibPRszOdXG78Vig_boKaVCD9A,2082
-biolib/cli/data_record.py,sha256=08JbZkFWKMo0PrnhhG0jQEKnNW7pPLti9cOw8s1TWfI,3344
+biolib/cli/data_record.py,sha256=t8DfJK2EZ_SNZ9drDA_N5Jqy8DNwf9f5SlFrIaOvtv0,3501
 biolib/cli/download_container.py,sha256=HIZVHOPmslGE5M2Dsp9r2cCkAEJx__vcsDz5Wt5LRos,483
 biolib/cli/init.py,sha256=wQOfii_au-d30Hp7DdH-WVw-WVraKvA_zY4za1w7DE8,821
 biolib/cli/lfs.py,sha256=z2qHUwink85mv9yDgifbVKkVwuyknGhMDTfly_gLKJM,4151
@@ -74,19 +75,19 @@ biolib/compute_node/job_worker/cache_state.py,sha256=MwjSRzcJJ_4jybqvBL4xdgnDYSI
 biolib/compute_node/job_worker/cache_types.py,sha256=ajpLy8i09QeQS9dEqTn3T6NVNMY_YsHQkSD5nvIHccQ,818
 biolib/compute_node/job_worker/docker_image_cache.py,sha256=ansHIkJIq_EMW1nZNlW-RRLVVeKWTbzNICYaOHpKiRE,7460
 biolib/compute_node/job_worker/executors/__init__.py,sha256=bW6t1qi3PZTlHM4quaTLa8EI4ALTCk83cqcVJfJfJfE,145
-biolib/compute_node/job_worker/executors/docker_executor.py,sha256=2H7GooL0oAifPcbie0unatB4fRoHyqbsr6S91uagc_g,27952
+biolib/compute_node/job_worker/executors/docker_executor.py,sha256=LcYc4x4-vkmRM1FN1SNj7xRj4qZUNVjQWtgbYMFg3JU,28029
 biolib/compute_node/job_worker/executors/docker_types.py,sha256=VhsU1DKtJjx_BbCkVmiPZPH4ROiL1ygW1Y_s1Kbpa2o,216
 biolib/compute_node/job_worker/executors/tars/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/compute_node/job_worker/executors/types.py,sha256=yP5gG39hr-DLnw9bOE--VHi-1arDbIYiGuV1rlTbbHI,1466
 biolib/compute_node/job_worker/job_legacy_input_wait_timeout_thread.py,sha256=_cvEiZbOwfkv6fYmfrvdi_FVviIEYr_dSClQcOQaUWM,1198
 biolib/compute_node/job_worker/job_max_runtime_timer_thread.py,sha256=K_xgz7IhiIjpLlXRk8sqaMyLoApcidJkgu29sJX0gb8,1174
-biolib/compute_node/job_worker/job_storage.py,sha256=LNkklckDLbYgCHsK5FGrEK75Kw-H4f4JcTCAtuE9His,4035
+biolib/compute_node/job_worker/job_storage.py,sha256=lScHI3ubcHKagSEW243tgbIWXUfbWDHDjEOPMvXxJE8,4603
 biolib/compute_node/job_worker/job_worker.py,sha256=fuWoYJo9HOqLmWl8yeCXh0mhT4ebbkrWac-BVb58khs,28842
 biolib/compute_node/job_worker/large_file_system.py,sha256=XXqRlVtYhs-Ji9zQGIk5KQPXFO_Q5jJH0nnlw4GkeMY,10461
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
 biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO33vWtG60xg,1282
-biolib/compute_node/remote_host_proxy.py,sha256=CNWJLXXYm8DGujxEJIsg1wUKFoSgU0nhwdmjMn5gelE,14690
+biolib/compute_node/remote_host_proxy.py,sha256=ibkWqvLJJHwOM7GzYqJV613QPacU5392vjnDK8BNJIU,15213
 biolib/compute_node/socker_listener_thread.py,sha256=T5_UikA3MB9bD5W_dckYLPTgixh72vKUlgbBvj9dbM0,1601
 biolib/compute_node/socket_sender_thread.py,sha256=YgamPHeUm2GjMFGx8qk-99WlZhEs-kAb3q_2O6qByig,971
 biolib/compute_node/utils.py,sha256=M7i_WTyxbFM3Lri9RWZ_8FeQNYrQIWpKGLfp2I55oeY,4677
@@ -95,7 +96,7 @@ biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLU
 biolib/compute_node/webserver/webserver.py,sha256=15PkRyhtdtSgFDxa0z78aPO4ciZURsFqJYi-HtUmZF8,6494
 biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
 biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
-biolib/compute_node/webserver/worker_thread.py,sha256=26tG73TADnOcXsAr7Iyf6smrLlCqB4x-vvmpUb8WqnA,11569
+biolib/compute_node/webserver/worker_thread.py,sha256=GRRBUqXdMKvbjyLQhYlqGIbFKeU2iiEXIe5IXi9wgdg,11806
 biolib/experiments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 biolib/experiments/experiment.py,sha256=jIRixmQm3Gq9YdJ3I0-rE1vFukXqq6U4zXehFOJ1yZk,7614
 biolib/jobs/__init__.py,sha256=aIb2H2DHjQbM2Bs-dysFijhwFcL58Blp0Co0gimED3w,32
@@ -116,8 +117,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=ZQFcaE37B2dtucN2zDjOmdya_X0ITc1zBFZJNQY13XA,5183
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.1.2193.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.1.2193.dist-info/METADATA,sha256=NCFsHsCnbnOhw1O2L-polZfFa2pNdtA89TfBW7f_o7E,1508
-pybiolib-1.1.2193.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-pybiolib-1.1.2193.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.1.2193.dist-info/RECORD,,
+pybiolib-1.2.7.dev0.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.7.dev0.dist-info/METADATA,sha256=O1duMBotGWLsk_tKeihafeOizbCNQVWjCVT7SbzVTZ4,1510
+pybiolib-1.2.7.dev0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+pybiolib-1.2.7.dev0.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.7.dev0.dist-info/RECORD,,