pybiolib 1.2.1111__py3-none-any.whl → 1.2.1117__py3-none-any.whl

biolib/compute_node/remote_host_proxy.py

@@ -4,9 +4,7 @@ import tarfile
 import time
 from urllib.parse import urlparse
 
-from docker.errors import ImageNotFound
 from docker.models.containers import Container
-from docker.models.images import Image
 from docker.models.networks import Network
 from docker.types import EndpointConfig
 
@@ -17,7 +15,8 @@ from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
-from biolib.typing_utils import Dict, List, Optional, cast
+from biolib.compute_node.webserver.proxy_utils import get_biolib_nginx_proxy_image
+from biolib.typing_utils import Dict, List, Optional
 
 
 # Prepare for remote hosts with specified port
@@ -83,7 +82,7 @@ class RemoteHostProxy:
             try:
                 self._container = docker.containers.create(
                     detach=True,
-                    image=self._get_biolib_remote_host_proxy_image(),
+                    image=get_biolib_nginx_proxy_image(),
                     name=self._name,
                     network=BIOLIB_PROXY_NETWORK_NAME,
                     networking_config=networking_config,
@@ -130,23 +129,6 @@ class RemoteHostProxy:
         if self._container:
             self._container.remove(force=True)
 
-    def _get_biolib_remote_host_proxy_image(self) -> Image:
-        if utils.IS_RUNNING_IN_CLOUD:
-            try:
-                logger_no_user_data.debug('Getting local Docker image for remote host proxy')
-                return cast(Image, self._docker.images.get('biolib-remote-host-proxy:latest'))
-            except ImageNotFound:
-                logger_no_user_data.debug(
-                    'Local Docker image for remote host proxy not available. Falling back to public image...'
-                )
-
-        public_image_uri = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
-        try:
-            logger_no_user_data.debug('Getting public Docker image for remote host proxy')
-            return cast(Image, self._docker.images.get(public_image_uri))
-        except ImageNotFound:
-            logger_no_user_data.debug('Pulling public Docker image for remote host proxy')
-            return cast(Image, self._docker.images.pull(public_image_uri))
 
     def _write_nginx_config_to_container(self, upstream_server_name: str, upstream_server_ports: List[int]) -> None:
         if not self._container:

biolib/compute_node/webserver/compute_node_results_proxy.py

@@ -0,0 +1,140 @@
+import io
+import tarfile
+import time
+
+from docker.models.containers import Container
+
+from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger_no_user_data
+from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
+from biolib.typing_utils import Optional
+from biolib.compute_node.webserver.proxy_utils import get_biolib_nginx_proxy_image
+
+class ComputeNodeResultsProxy:
+    _instance: Optional['ComputeNodeResultsProxy'] = None
+
+    def __init__(self, tls_pem_certificate_path: str, tls_pem_key_path: str):
+        assert tls_pem_certificate_path, "tls_pem_certificate_path is required"
+        assert tls_pem_key_path, "tls_pem_key_path is required"
+        self._name = 'biolib-compute-node-results-proxy'
+        self._container: Optional[Container] = None
+        self._docker = BiolibDockerClient().get_docker_client()
+        self._tls_pem_certificate_path = tls_pem_certificate_path
+        self._tls_pem_key_path = tls_pem_key_path
+
+    @staticmethod
+    def start_proxy(tls_pem_certificate_path: str, tls_pem_key_path: str) -> None:
+        if ComputeNodeResultsProxy._instance is None:
+            ComputeNodeResultsProxy._instance = ComputeNodeResultsProxy(tls_pem_certificate_path, tls_pem_key_path)
+            ComputeNodeResultsProxy._instance._start()  # pylint: disable=protected-access
+
+    @staticmethod
+    def stop_proxy() -> None:
+        if ComputeNodeResultsProxy._instance is not None:
+            ComputeNodeResultsProxy._instance._terminate()  # pylint: disable=protected-access
+            ComputeNodeResultsProxy._instance = None
+
+    def _start(self) -> None:
+        docker = BiolibDockerClient.get_docker_client()
+
+        for index in range(3):
+            logger_no_user_data.debug(
+                f'Attempt {index} at creating ComputeNodeResultsProxy container "{self._name}"...'
+            )
+            try:
+                self._container = docker.containers.create(
+                    detach=True,
+                    image=get_biolib_nginx_proxy_image(),
+                    name=self._name,
+                    network=BIOLIB_PROXY_NETWORK_NAME,
+                    ports={'443/tcp': 20443},
+                    volumes={
+                        self._tls_pem_certificate_path: {'bind': '/etc/ssl/certs/certificate.pem', 'mode': 'ro'},
+                        self._tls_pem_key_path: {'bind': '/etc/ssl/private/private_key.pem', 'mode': 'ro'},
+                    },
+                )
+                break
+            except Exception as error:
+                logger_no_user_data.exception(f'Failed to create container "{self._name}" hit error: {error}')
+
+            logger_no_user_data.debug('Sleeping before re-trying container creation...')
+            time.sleep(3)
+
+        if not self._container or not self._container.id:
+            raise BioLibError(f'Exceeded re-try limit for creating container {self._name}')
+
+        self._write_nginx_config_to_container()
+        self._container.start()
+
+        proxy_is_ready = False
+        for retry_count in range(1, 5):
+            time.sleep(0.5 * retry_count)
+            container_logs = self._container.logs()
+            if b'ready for start up\n' in container_logs or b'start worker process ' in container_logs:
+                proxy_is_ready = True
+                break
+
+        if not proxy_is_ready:
+            self._terminate()
+            raise Exception('ComputeNodeResultsProxy did not start properly')
+
+        self._container.reload()
+
+    def _terminate(self):
+        if self._container:
+            self._container.remove(force=True)
+
+
+    def _write_nginx_config_to_container(self) -> None:
+        if not self._container:
+            raise Exception('ComputeNodeResultsProxy container not defined when attempting to write NGINX config')
+
+        docker = BiolibDockerClient.get_docker_client()
+
+        nginx_config = """
+events {
+  worker_connections  1024;
+}
+
+http {
+    client_max_body_size 0;
+    resolver 127.0.0.11 ipv6=off valid=30s;
+
+    server {
+        listen       443 ssl;
+        ssl_certificate /etc/ssl/certs/certificate.pem;
+        ssl_certificate_key /etc/ssl/private/private_key.pem;
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
+        ssl_prefer_server_ciphers on;
+
+        location / {
+            if ($http_biolib_result_uuid = "") {
+                return 400 "Missing biolib-result-uuid header";
+            }
+
+            if ($http_biolib_result_port = "") {
+                return 400 "Missing biolib-result-port header";
+            }
+
+            set $target_host "biolib-app-caller-proxy-$http_biolib_result_uuid";
+            proxy_pass http://$target_host:1080;
+            proxy_set_header biolib-result-uuid $http_biolib_result_uuid;
+            proxy_set_header biolib-result-port $http_biolib_result_port;
+            proxy_pass_request_headers on;
+        }
+    }
+}
+"""
+
+        nginx_config_bytes = nginx_config.encode()
+        tarfile_in_memory = io.BytesIO()
+        with tarfile.open(fileobj=tarfile_in_memory, mode='w:gz') as tar:
+            info = tarfile.TarInfo('/nginx.conf')
+            info.size = len(nginx_config_bytes)
+            tar.addfile(info, io.BytesIO(nginx_config_bytes))
+
+        tarfile_bytes = tarfile_in_memory.getvalue()
+        tarfile_in_memory.close()
+        docker.api.put_archive(self._container.id, '/etc/nginx', tarfile_bytes)

biolib/compute_node/webserver/proxy_utils.py

@@ -0,0 +1,28 @@
+from docker.errors import ImageNotFound
+from docker.models.images import Image
+
+from biolib import utils
+from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_logging import logger_no_user_data
+from biolib.typing_utils import cast
+
+
+def get_biolib_nginx_proxy_image() -> Image:
+    docker = BiolibDockerClient().get_docker_client()
+
+    if utils.IS_RUNNING_IN_CLOUD:
+        try:
+            logger_no_user_data.debug('Getting local Docker image for nginx proxy')
+            return cast(Image, docker.images.get('biolib-remote-host-proxy:latest'))
+        except ImageNotFound:
+            logger_no_user_data.debug(
+                'Local Docker image for nginx proxy not available. Falling back to public image...'
+            )
+
+    public_image_uri = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
+    try:
+        logger_no_user_data.debug('Getting public Docker image for nginx proxy')
+        return cast(Image, docker.images.get(public_image_uri))
+    except ImageNotFound:
+        logger_no_user_data.debug('Pulling public Docker image for nginx proxy')
+        return cast(Image, docker.images.pull(public_image_uri))

biolib/compute_node/webserver/webserver.py

@@ -17,6 +17,7 @@ from biolib.biolib_logging import TRACE, logger, logger_no_user_data
 from biolib.compute_node.cloud_utils.cloud_utils import CloudUtils
 from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
 from biolib.compute_node.webserver import webserver_utils
+from biolib.compute_node.webserver.compute_node_results_proxy import ComputeNodeResultsProxy
 from biolib.compute_node.webserver.gunicorn_flask_application import GunicornFlaskApplication
 from biolib.compute_node.webserver.webserver_utils import get_job_compute_state_or_404
 from biolib.typing_utils import Optional
@@ -152,6 +153,12 @@ def start_webserver(
             time.sleep(2)
 
         if utils.IS_RUNNING_IN_CLOUD:
+            try:
+                logger_no_user_data.debug('Stopping ComputeNodeResultsProxy')
+                ComputeNodeResultsProxy.stop_proxy()
+            except BaseException:
+                logger_no_user_data.exception('Failed to stop ComputeNodeResultsProxy')
+
             try:
                 logger_no_user_data.debug(f'Removing Docker network {BIOLIB_PROXY_NETWORK_NAME}')
                 docker_client = BiolibDockerClient.get_docker_client()
@@ -184,6 +191,12 @@ def start_webserver(
                 logger_no_user_data.exception(f'Failed to create Docker network {BIOLIB_PROXY_NETWORK_NAME}')
 
             if biolib_proxy_network:
+                try:
+                    logger_no_user_data.debug('Starting ComputeNodeResultsProxy')
+                    ComputeNodeResultsProxy.start_proxy(tls_pem_certificate_path, tls_pem_key_path)
+                except BaseException:
+                    logger_no_user_data.exception('Failed to start ComputeNodeResultsProxy')
+
                 CloudUtils.initialize()
                 webserver_utils.start_auto_shutdown_timer()
             else:

{pybiolib-1.2.1111.dist-info → pybiolib-1.2.1117.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybiolib
-Version: 1.2.1111
+Version: 1.2.1117
 Summary: BioLib Python Client
 License: MIT
 Keywords: biolib

{pybiolib-1.2.1111.dist-info → pybiolib-1.2.1117.dist-info}/RECORD

@@ -126,13 +126,15 @@ biolib/compute_node/job_worker/large_file_system.py,sha256=Xe_LILVfTD9LXb-0HwLqG
 biolib/compute_node/job_worker/mappings.py,sha256=Z48Kg4nbcOvsT2-9o3RRikBkqflgO4XeaWxTGz-CNvI,2499
 biolib/compute_node/job_worker/utilization_reporter_thread.py,sha256=7tm5Yk9coqJ9VbEdnO86tSXI0iM0omwIyKENxdxiVXk,8575
 biolib/compute_node/job_worker/utils.py,sha256=wgxcIA8yAhUPdCwyvuuJ0JmreyWmmUoBO33vWtG60xg,1282
-biolib/compute_node/remote_host_proxy.py,sha256=xyjq0zKSxbJGUA6a0q9JCnzquk7gIztbI1clLGnymQM,17278
+biolib/compute_node/remote_host_proxy.py,sha256=W0zzPtiHAOofCslkn9cnjFY1VKEbMSdrer87j23Zv_8,16295
 biolib/compute_node/socker_listener_thread.py,sha256=T5_UikA3MB9bD5W_dckYLPTgixh72vKUlgbBvj9dbM0,1601
 biolib/compute_node/socket_sender_thread.py,sha256=YgamPHeUm2GjMFGx8qk-99WlZhEs-kAb3q_2O6qByig,971
 biolib/compute_node/utils.py,sha256=fvdbetPKMdfHkPqNZRw6eln_i13myu-n8tuceTUcfPU,4913
 biolib/compute_node/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+biolib/compute_node/webserver/compute_node_results_proxy.py,sha256=sP_ZzjJjAusrHzCQC3qu3ItGY3YZzPinaFOHbattDjs,5565
 biolib/compute_node/webserver/gunicorn_flask_application.py,sha256=jPfR_YvNBekLUXWo_vHFV-FIwlb8s8tacKmGHvh93qc,914
-biolib/compute_node/webserver/webserver.py,sha256=Ec5ZgOUBb_1pINjHrlENMUpLdAPiBiTyiaP51-FZ4gQ,8362
+biolib/compute_node/webserver/proxy_utils.py,sha256=JNt1oqPuR5n63E5NC1l8AZN51NLuMw4lqVkocZnq72o,1164
+biolib/compute_node/webserver/webserver.py,sha256=XOLYIJcrs_MU3uvct7SGPn8cYDi8ugA_wkDo6JeOQDU,9061
 biolib/compute_node/webserver/webserver_types.py,sha256=2t8EaFKESnves3BA_NBdnS2yAdo1qwamCFHiSt888nE,380
 biolib/compute_node/webserver/webserver_utils.py,sha256=XWvwYPbWNR3qS0FYbLLp-MDDfVk0QdaAmg3xPrT0H2s,4234
 biolib/compute_node/webserver/worker_thread.py,sha256=7uD9yQPhePYvP2HCJ27EeZ_h6psfIWFgqm1RHZxzobs,12483
@@ -155,8 +157,8 @@ biolib/utils/cache_state.py,sha256=u256F37QSRIVwqKlbnCyzAX4EMI-kl6Dwu6qwj-Qmag,3
 biolib/utils/multipart_uploader.py,sha256=XvGP1I8tQuKhAH-QugPRoEsCi9qvbRk-DVBs5PNwwJo,8452
 biolib/utils/seq_util.py,sha256=Ozk0blGtPur_D9MwShD02r_mphyQmgZkx-lOHOwnlIM,6730
 biolib/utils/zip/remote_zip.py,sha256=0wErYlxir5921agfFeV1xVjf29l9VNgGQvNlWOlj2Yc,23232
-pybiolib-1.2.1111.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
-pybiolib-1.2.1111.dist-info/METADATA,sha256=FimvlfOuyzXFZZTJy_czDvIwLdbiL6cg0PATItgsgho,1571
-pybiolib-1.2.1111.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-pybiolib-1.2.1111.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
-pybiolib-1.2.1111.dist-info/RECORD,,
+pybiolib-1.2.1117.dist-info/LICENSE,sha256=F2h7gf8i0agDIeWoBPXDMYScvQOz02pAWkKhTGOHaaw,1067
+pybiolib-1.2.1117.dist-info/METADATA,sha256=SfoHQrH4vo-GUXZYGyZdjFpwou9KjMBwUH7bIVwA7sM,1571
+pybiolib-1.2.1117.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+pybiolib-1.2.1117.dist-info/entry_points.txt,sha256=p6DyaP_2kctxegTX23WBznnrDi4mz6gx04O5uKtRDXg,42
+pybiolib-1.2.1117.dist-info/RECORD,,