pybiolib 1.1.1881__py3-none-any.whl → 1.2.7.dev0__py3-none-any.whl

biolib/compute_node/remote_host_proxy.py

@@ -1,21 +1,22 @@
+import base64
 import io
-import tarfile
 import subprocess
+import tarfile
 import time
+from urllib.parse import urlparse
 
-from docker.models.containers import Container  # type: ignore
 from docker.errors import ImageNotFound  # type: ignore
+from docker.models.containers import Container  # type: ignore
 from docker.models.images import Image  # type: ignore
 from docker.models.networks import Network  # type: ignore
 
 from biolib import utils
-from biolib.biolib_errors import BioLibError
-from biolib.compute_node.cloud_utils import CloudUtils
-from biolib.typing_utils import Optional, List
-from biolib.biolib_api_client import RemoteHost
+from biolib.biolib_api_client import BiolibApiClient, RemoteHost
 from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger_no_user_data
-from biolib.biolib_api_client import BiolibApiClient
+from biolib.compute_node.cloud_utils import CloudUtils
+from biolib.typing_utils import List, Optional
 
 
 # Prepare for remote hosts with specified port
@@ -24,31 +25,23 @@ class RemoteHostExtended(RemoteHost):
 
 
 class RemoteHostProxy:
-
     def __init__(
-            self,
-            remote_host: RemoteHost,
-            public_network: Network,
-            internal_network: Optional[Network],
-            job_id: str,
-            ports: List[int],
-            can_push_data_record_for_user: bool,
+        self,
+        remote_host: RemoteHost,
+        public_network: Network,
+        internal_network: Optional[Network],
+        job_id: str,
+        ports: List[int],
     ):
-        self._can_push_data_record_for_user: bool = can_push_data_record_for_user
         self.is_app_caller_proxy = remote_host['hostname'] == 'AppCallerProxy'
-
-        # Default to port 443 for now until backend serves remote_hosts with port specified
-        self._remote_host: RemoteHostExtended = RemoteHostExtended(
-            hostname=remote_host['hostname'],
-            ports=ports
-        )
+        self._remote_host: RemoteHostExtended = RemoteHostExtended(hostname=remote_host['hostname'], ports=ports)
         self._public_network: Network = public_network
         self._internal_network: Optional[Network] = internal_network
 
         if not job_id:
             raise Exception('RemoteHostProxy missing argument "job_id"')
 
-        self._name = f"biolib-remote-host-proxy-{job_id}-{self.hostname}"
+        self._name = f'biolib-remote-host-proxy-{job_id}-{self.hostname}'
         self._job_uuid = job_id
         self._container: Optional[Container] = None
         self._enclave_traffic_forwarder_processes: List[subprocess.Popen] = []
@@ -154,32 +147,24 @@ class RemoteHostProxy:
             raise Exception('RemoteHostProxy container not defined when attempting to write NGINX config')
 
         docker = BiolibDockerClient.get_docker_client()
-        base_url = BiolibApiClient.get().base_url
+        upstream_hostname = urlparse(BiolibApiClient.get().base_url).hostname
         if self.is_app_caller_proxy:
+            if not utils.IS_RUNNING_IN_CLOUD:
+                raise BioLibError('Calling apps inside apps is not supported in local compute environment')
+
             logger_no_user_data.debug(f'Job "{self._job_uuid}" writing config for and starting App Caller Proxy...')
-            if utils.BIOLIB_CLOUD_BASE_URL:
-                cloud_base_url = utils.BIOLIB_CLOUD_BASE_URL
-            else:
-                if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
-                    cloud_base_url = 'https://biolibcloud.com'
-                else:
-                    raise BioLibError('Calling apps inside apps is not supported in local compute environment')
-
-            if utils.IS_RUNNING_IN_CLOUD:
-                config = CloudUtils.get_webserver_config()
-                s3_results_bucket_name = config['s3_general_storage_bucket_name']
-                s3_results_base_url = f'https://{s3_results_bucket_name}.s3.amazonaws.com'
-            else:
-                if base_url in ('https://biolib.com', 'https://staging-elb.biolib.com'):
-                    s3_results_base_url = 'https://biolib-cloud-api.s3.amazonaws.com'
-                else:
-                    raise BioLibError("Calling apps inside apps locally is only supported on biolib.com")
+            config = CloudUtils.get_webserver_config()
+            compute_node_uuid = config['compute_node_info']['public_id']
+            compute_node_auth_token = config['compute_node_info']['auth_token']
 
             # TODO: Get access_token from new API class instead
             access_token = BiolibApiClient.get().access_token
             bearer_token = f'Bearer {access_token}' if access_token else ''
 
-            nginx_config = f'''
+            biolib_index_basic_auth = f'compute_node|admin:{compute_node_auth_token},{self._job_uuid}'
+            biolib_index_basic_auth_base64 = base64.b64encode(biolib_index_basic_auth.encode('utf-8')).decode('utf-8')
+
+            nginx_config = f"""
 events {{
   worker_connections  1024;
 }}
@@ -196,29 +181,19 @@ http {{
         default    "";
     }}
 
-    map $request_method $bearer_token_on_patch {{
-        PATCH      "{bearer_token}";
-        default    "";
-    }}
-
     map $request_method $bearer_token_on_patch_and_get {{
         PATCH      "{bearer_token}";
         GET        "{bearer_token}";
         default    "";
     }}
 
-    map $request_method $bearer_token_on_post_and_get {{
-        POST       "{bearer_token}";
-        GET        "{bearer_token}";
-        default    "";
-    }}
-
     server {{
         listen       80;
-        resolver 127.0.0.11 valid=30s;
+        resolver 127.0.0.11 ipv6=off valid=30s;
+        set $upstream_hostname {upstream_hostname};
 
         location ~* "^/api/jobs/cloud/(?<job_id>[a-z0-9-]{{36}})/status/$" {{
-            proxy_pass               {base_url}/api/jobs/cloud/$job_id/status/;
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$job_id/status/;
             proxy_set_header         authorization $bearer_token_on_get;
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
@@ -226,35 +201,35 @@ http {{
 
         location ~* "^/api/jobs/cloud/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/cloud/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/start_upload/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/start_upload/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/start_upload/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/presigned_upload_url/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/presigned_upload_url/$is_args$args;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/presigned_upload_url/$is_args$args;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/complete_upload/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/storage/input/complete_upload/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/complete_upload/;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
         
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/main_result/$" {{
-            proxy_pass                  {base_url}/api/jobs/$job_id/main_result/;
+            proxy_pass                  https://$upstream_hostname/api/jobs/$job_id/main_result/;
             proxy_set_header            authorization "";
             proxy_set_header            cookie "";
             proxy_pass_request_headers  on;
@@ -262,7 +237,7 @@ http {{
         }}
 
         location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/$" {{
-            proxy_pass               {base_url}/api/jobs/$job_id/;
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/;
             proxy_set_header         authorization $bearer_token_on_patch_and_get;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
@@ -271,7 +246,7 @@ http {{
 
         location ~* "^/api/jobs/create_job_with_data/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/create_job_with_data/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/create_job_with_data/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
@@ -280,78 +255,91 @@ http {{
 
         location ~* "^/api/jobs/$" {{
             # Note: Using $1 here as URI part from regex must be used for proxy_pass
-            proxy_pass               {base_url}/api/jobs/$1;
+            proxy_pass               https://$upstream_hostname/api/jobs/$1;
             proxy_set_header         authorization $bearer_token_on_post;
             proxy_set_header         caller-job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /api/lfs/ {{
-            proxy_pass               {base_url}/api/lfs/;
-            proxy_set_header         authorization {'$bearer_token_on_post_and_get' if self._can_push_data_record_for_user else '""'};
+        location ~ "^/api/jobs/{self._job_uuid}/notes/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/{self._job_uuid}/notes/$1;
+            proxy_set_header         authorization "";
+            proxy_set_header         job-auth-token "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         compute-node-uuid "{compute_node_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /api/ {{
-            proxy_pass               {base_url}/api/;
+        location /api/lfs/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /cloud-proxy/ {{
-            proxy_pass               {cloud_base_url}/cloud-proxy/;
+        location /api/app/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        location /job-storage/ {{
-            proxy_pass               {s3_results_base_url}/job-storage/;
+        location /api/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
         location /proxy/storage/job-storage/ {{
-            proxy_pass               {cloud_base_url}/proxy/storage/job-storage/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
-        {f"""
         location /proxy/storage/lfs/versions/ {{
-            proxy_pass               {cloud_base_url}/proxy/storage/lfs/versions/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
-        """ if self._can_push_data_record_for_user else ''}
 
         location /proxy/cloud/ {{
-            proxy_pass               {cloud_base_url}/proxy/cloud/;
+            proxy_pass               https://$upstream_hostname$request_uri;
             proxy_set_header         authorization "";
             proxy_set_header         cookie "";
             proxy_ssl_server_name    on;
         }}
 
+        location /proxy/index/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "Basic {biolib_index_basic_auth_base64}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
         location / {{
             return 404 "Not found";
         }}
     }}
 }}
-'''
+"""
         else:
-            nginx_config = '''
+            nginx_config = """
 events {}
 error_log /dev/stdout info;
 stream {
-    resolver 127.0.0.11 valid=30s;'''
+    resolver 127.0.0.11 valid=30s;"""
             for idx, upstream_server_port in enumerate(upstream_server_ports):
-                nginx_config += f'''
+                nginx_config += f"""
     map "" $upstream_{idx} {{
         default {upstream_server_name}:{upstream_server_port};
     }}
@@ -364,11 +352,11 @@ stream {
     server {{
         listen          {self._remote_host['ports'][idx]} udp;
         proxy_pass      $upstream_{idx};
-    }}'''
+    }}"""
 
-            nginx_config += '''
+            nginx_config += """
 }
-'''
+"""
 
         nginx_config_bytes = nginx_config.encode()
         tarfile_in_memory = io.BytesIO()

biolib/compute_node/webserver/webserver_types.py

@@ -16,5 +16,4 @@ class WebserverConfig(TypedDict):
     base_url: str
     compute_node_info: ComputeNodeInfo
     is_dev: bool
-    s3_general_storage_bucket_name: str
     shutdown_times: ShutdownTimes

biolib/compute_node/webserver/worker_thread.py

@@ -2,23 +2,23 @@ import base64
 import os
 import random
 import shutil
+import socket
 import sys
-import time
 import threading
-import socket
+import time
 from queue import Queue
 
-from biolib import utils
+from biolib import api, utils
+from biolib.biolib_binary_format import ModuleOutputV2, SystemException, SystemStatusUpdate
 from biolib.biolib_binary_format.utils import LocalFileIndexableBuffer
+from biolib.biolib_logging import logger, logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.job_worker import JobWorkerProcess
 from biolib.compute_node.job_worker.job_storage import JobStorage
 from biolib.compute_node.socker_listener_thread import SocketListenerThread
 from biolib.compute_node.socket_sender_thread import SocketSenderThread
+from biolib.compute_node.utils import SystemExceptionCodes, WorkerThreadException, get_package_type
 from biolib.compute_node.webserver import webserver_utils
-from biolib.biolib_binary_format import SystemStatusUpdate, SystemException, ModuleOutputV2
-from biolib.compute_node.utils import get_package_type, WorkerThreadException, SystemExceptionCodes
-from biolib.biolib_logging import logger, logger_no_user_data
 
 SOCKET_HOST = '127.0.0.1'
 
@@ -37,7 +37,7 @@ class WorkerThread(threading.Thread):
             self._sender_thread = None
             self._start_and_connect_to_compute_process()
 
-            logger.debug(f"WorkerThread connected to port {self._socket_port}")
+            logger.debug(f'WorkerThread connected to port {self._socket_port}')
 
         except Exception as exception:
             logger_no_user_data.error(exception)
@@ -79,20 +79,16 @@ class WorkerThread(threading.Thread):
                     if progress == 94:
                         # Get Job exit code
                         try:
-                            module_output_path = os.path.join(self._job_temporary_dir,
-                                                              JobStorage.module_output_file_name)
-                            module_output = ModuleOutputV2(
-                                buffer=LocalFileIndexableBuffer(
-                                    filename=module_output_path
-                                )
+                            module_output_path = os.path.join(
+                                self._job_temporary_dir,
+                                JobStorage.module_output_file_name,
                             )
+                            module_output = ModuleOutputV2(buffer=LocalFileIndexableBuffer(filename=module_output_path))
                             self.compute_state['exit_code'] = module_output.get_exit_code()
                             logger_no_user_data.debug(f"Got exit code: {self.compute_state['exit_code']}")
 
                         except Exception as error:  # pylint: disable=broad-except
-                            logger_no_user_data.error(
-                                f'Could not get exit_code from module output due to: {error}'
-                            )
+                            logger_no_user_data.error(f'Could not get exit_code from module output due to: {error}')
 
                         if utils.IS_RUNNING_IN_CLOUD:
                             JobStorage.upload_module_output(
@@ -107,7 +103,7 @@ class WorkerThread(threading.Thread):
                 elif package_type == 'SystemException':
                     error_code = SystemException(package).deserialize()
                     self.compute_state['status']['error_code'] = error_code
-                    logger.debug("Hit error. Terminating Worker Thread and Compute Process")
+                    logger.debug('Hit error. Terminating Worker Thread and Compute Process')
                     self.compute_state['progress'] = 95
                     self.terminate()
 
@@ -153,10 +149,10 @@ class WorkerThread(threading.Thread):
 
         # Starting a thread for accepting connections before starting the process that should to connect to the socket
         logger_no_user_data.debug('Starting connection thread')
-        self._connection_thread = threading.Thread(target=self._accept_new_socket_connection, args=[
-            received_messages_queue,
-            messages_to_send_queue
-        ])
+        self._connection_thread = threading.Thread(
+            target=self._accept_new_socket_connection,
+            args=[received_messages_queue, messages_to_send_queue],
+        )
         self._connection_thread.start()
         logger_no_user_data.debug('Started connection thread')
         logger_no_user_data.debug('Starting compute process')
@@ -177,6 +173,16 @@ class WorkerThread(threading.Thread):
         self._sender_thread.start()
 
     def terminate(self) -> None:
+        cloud_job_uuid = self.compute_state['cloud_job_id']
+        exit_code = self.compute_state.get('exit_code')
+        system_exception_code = self.compute_state['status'].get('error_code')
+        if utils.IS_RUNNING_IN_CLOUD:
+            CloudUtils.finish_cloud_job(
+                cloud_job_id=cloud_job_uuid,
+                system_exception_code=system_exception_code,
+                exit_code=exit_code,
+            )
+
         deregistered_due_to_error = False
         if self._job_worker_process:
             logger_no_user_data.debug(
@@ -184,7 +190,8 @@ class WorkerThread(threading.Thread):
             )
             self._job_worker_process.terminate()
 
-            for _ in range(10):
+            clean_up_timeout_in_seconds = 600
+            for _ in range(clean_up_timeout_in_seconds):
                 if self._job_worker_process.exitcode is not None:
                     logger_no_user_data.debug(
                         f'Job "{self._job_uuid}" worker process exitcode {self._job_worker_process.exitcode}'
@@ -196,28 +203,18 @@ class WorkerThread(threading.Thread):
 
             if self._job_worker_process.exitcode is None:
                 # TODO: Figure out if more error handling is necessary here
-                logger_no_user_data.error(f'Job {self._job_uuid} worker process did not exit within 10 seconds')
+                logger_no_user_data.error(
+                    f'Job {self._job_uuid} worker process did not exit within {clean_up_timeout_in_seconds} seconds'
+                )
                 if utils.IS_RUNNING_IN_CLOUD:
                     logger_no_user_data.error('Deregistering compute node...')
                     CloudUtils.deregister(error='job_cleanup_timed_out')
                     deregistered_due_to_error = True
 
         # Delete result as error occurred
-        system_exception_code = self.compute_state['status'].get('error_code')
         if system_exception_code and os.path.exists(self._job_temporary_dir):
             shutil.rmtree(self._job_temporary_dir)
 
-        exit_code = self.compute_state.get('exit_code')
-
-        if utils.IS_RUNNING_IN_CLOUD:
-            # Get and send compute node exception code and job exit code if present
-            logger_no_user_data.debug(f"Sending exit code {exit_code}")
-            CloudUtils.finish_cloud_job(
-                cloud_job_id=self.compute_state['cloud_job_id'],
-                system_exception_code=system_exception_code,
-                exit_code=exit_code
-            )
-
         if self._socket:
             self._socket.close()
 
@@ -225,7 +222,7 @@ class WorkerThread(threading.Thread):
             self._connection.close()
 
         if self.compute_state['progress'] == 95:
-            seconds_to_sleep = 60  # 1 minute
+            seconds_to_sleep = 5
             logger_no_user_data.debug(
                 f'Job "{self._job_uuid}" worker thread sleeping for {seconds_to_sleep} seconds before cleaning up'
             )
@@ -234,7 +231,7 @@ class WorkerThread(threading.Thread):
 
         compute_state_dict = webserver_utils.JOB_ID_TO_COMPUTE_STATE_DICT
         if self._job_uuid in compute_state_dict:
-            # Delete result as user has not started download within 60 seconds
+            # Delete result as user has not started download
             if compute_state_dict[self._job_uuid]['progress'] == 95 and os.path.exists(self._job_temporary_dir):
                 shutil.rmtree(self._job_temporary_dir)
 
@@ -245,12 +242,18 @@ class WorkerThread(threading.Thread):
                 f'Job "{self._job_uuid}" could not be found, maybe it has already been cleaned up'
             )
 
-        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread terminated')
-
         if utils.IS_RUNNING_IN_CLOUD:
+            config = CloudUtils.get_webserver_config()
+            logger_no_user_data.debug(f'Job "{self._job_uuid}" reporting CloudJob "{cloud_job_uuid}" as cleaned up...')
+            api.client.post(
+                path=f'/internal/compute-nodes/cloud-jobs/{cloud_job_uuid}/cleaned-up/',
+                headers={'Compute-Node-Auth-Token': config['compute_node_info']['auth_token']},
+            )
+
             if deregistered_due_to_error:
                 CloudUtils.shutdown()  # shutdown now
             else:
                 webserver_utils.update_auto_shutdown_time()
 
+        logger_no_user_data.debug(f'Job "{self._job_uuid}" worker thread exiting...')
         sys.exit()