pybiolib 1.2.1111__tar.gz → 1.2.1117__tar.gz

{pybiolib-1.2.1111 → pybiolib-1.2.1117}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pybiolib
-Version: 1.2.1111
+Version: 1.2.1117
 Summary: BioLib Python Client
 License: MIT
 Keywords: biolib

{pybiolib-1.2.1111 → pybiolib-1.2.1117}/biolib/compute_node/remote_host_proxy.py

@@ -4,9 +4,7 @@ import tarfile
 import time
 from urllib.parse import urlparse
 
-from docker.errors import ImageNotFound
 from docker.models.containers import Container
-from docker.models.images import Image
 from docker.models.networks import Network
 from docker.types import EndpointConfig
 
@@ -17,7 +15,8 @@ from biolib.biolib_errors import BioLibError
 from biolib.biolib_logging import logger_no_user_data
 from biolib.compute_node.cloud_utils import CloudUtils
 from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
-from biolib.typing_utils import Dict, List, Optional, cast
+from biolib.compute_node.webserver.proxy_utils import get_biolib_nginx_proxy_image
+from biolib.typing_utils import Dict, List, Optional
 
 
 # Prepare for remote hosts with specified port
@@ -83,7 +82,7 @@ class RemoteHostProxy:
             try:
                 self._container = docker.containers.create(
                     detach=True,
-                    image=self._get_biolib_remote_host_proxy_image(),
+                    image=get_biolib_nginx_proxy_image(),
                     name=self._name,
                     network=BIOLIB_PROXY_NETWORK_NAME,
                     networking_config=networking_config,
@@ -130,23 +129,6 @@ class RemoteHostProxy:
         if self._container:
             self._container.remove(force=True)
 
-    def _get_biolib_remote_host_proxy_image(self) -> Image:
-        if utils.IS_RUNNING_IN_CLOUD:
-            try:
-                logger_no_user_data.debug('Getting local Docker image for remote host proxy')
-                return cast(Image, self._docker.images.get('biolib-remote-host-proxy:latest'))
-            except ImageNotFound:
-                logger_no_user_data.debug(
-                    'Local Docker image for remote host proxy not available. Falling back to public image...'
-                )
-
-        public_image_uri = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
-        try:
-            logger_no_user_data.debug('Getting public Docker image for remote host proxy')
-            return cast(Image, self._docker.images.get(public_image_uri))
-        except ImageNotFound:
-            logger_no_user_data.debug('Pulling public Docker image for remote host proxy')
-            return cast(Image, self._docker.images.pull(public_image_uri))
 
     def _write_nginx_config_to_container(self, upstream_server_name: str, upstream_server_ports: List[int]) -> None:
         if not self._container:

pybiolib-1.2.1117/biolib/compute_node/webserver/compute_node_results_proxy.py

@@ -0,0 +1,140 @@
+import io
+import tarfile
+import time
+
+from docker.models.containers import Container
+
+from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger_no_user_data
+from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
+from biolib.typing_utils import Optional
+from biolib.compute_node.webserver.proxy_utils import get_biolib_nginx_proxy_image
+
+class ComputeNodeResultsProxy:
+    _instance: Optional['ComputeNodeResultsProxy'] = None
+
+    def __init__(self, tls_pem_certificate_path: str, tls_pem_key_path: str):
+        assert tls_pem_certificate_path, "tls_pem_certificate_path is required"
+        assert tls_pem_key_path, "tls_pem_key_path is required"
+        self._name = 'biolib-compute-node-results-proxy'
+        self._container: Optional[Container] = None
+        self._docker = BiolibDockerClient().get_docker_client()
+        self._tls_pem_certificate_path = tls_pem_certificate_path
+        self._tls_pem_key_path = tls_pem_key_path
+
+    @staticmethod
+    def start_proxy(tls_pem_certificate_path: str, tls_pem_key_path: str) -> None:
+        if ComputeNodeResultsProxy._instance is None:
+            ComputeNodeResultsProxy._instance = ComputeNodeResultsProxy(tls_pem_certificate_path, tls_pem_key_path)
+            ComputeNodeResultsProxy._instance._start()  # pylint: disable=protected-access
+
+    @staticmethod
+    def stop_proxy() -> None:
+        if ComputeNodeResultsProxy._instance is not None:
+            ComputeNodeResultsProxy._instance._terminate()  # pylint: disable=protected-access
+            ComputeNodeResultsProxy._instance = None
+
+    def _start(self) -> None:
+        docker = BiolibDockerClient.get_docker_client()
+
+        for index in range(3):
+            logger_no_user_data.debug(
+                f'Attempt {index} at creating ComputeNodeResultsProxy container "{self._name}"...'
+            )
+            try:
+                self._container = docker.containers.create(
+                    detach=True,
+                    image=get_biolib_nginx_proxy_image(),
+                    name=self._name,
+                    network=BIOLIB_PROXY_NETWORK_NAME,
+                    ports={'443/tcp': 20443},
+                    volumes={
+                        self._tls_pem_certificate_path: {'bind': '/etc/ssl/certs/certificate.pem', 'mode': 'ro'},
+                        self._tls_pem_key_path: {'bind': '/etc/ssl/private/private_key.pem', 'mode': 'ro'},
+                    },
+                )
+                break
+            except Exception as error:
+                logger_no_user_data.exception(f'Failed to create container "{self._name}" hit error: {error}')
+
+            logger_no_user_data.debug('Sleeping before re-trying container creation...')
+            time.sleep(3)
+
+        if not self._container or not self._container.id:
+            raise BioLibError(f'Exceeded re-try limit for creating container {self._name}')
+
+        self._write_nginx_config_to_container()
+        self._container.start()
+
+        proxy_is_ready = False
+        for retry_count in range(1, 5):
+            time.sleep(0.5 * retry_count)
+            container_logs = self._container.logs()
+            if b'ready for start up\n' in container_logs or b'start worker process ' in container_logs:
+                proxy_is_ready = True
+                break
+
+        if not proxy_is_ready:
+            self._terminate()
+            raise Exception('ComputeNodeResultsProxy did not start properly')
+
+        self._container.reload()
+
+    def _terminate(self):
+        if self._container:
+            self._container.remove(force=True)
+
+
+    def _write_nginx_config_to_container(self) -> None:
+        if not self._container:
+            raise Exception('ComputeNodeResultsProxy container not defined when attempting to write NGINX config')
+
+        docker = BiolibDockerClient.get_docker_client()
+
+        nginx_config = """
+events {
+  worker_connections  1024;
+}
+
+http {
+    client_max_body_size 0;
+    resolver 127.0.0.11 ipv6=off valid=30s;
+
+    server {
+        listen       443 ssl;
+        ssl_certificate /etc/ssl/certs/certificate.pem;
+        ssl_certificate_key /etc/ssl/private/private_key.pem;
+        ssl_protocols TLSv1.2 TLSv1.3;
+        ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384;
+        ssl_prefer_server_ciphers on;
+
+        location / {
+            if ($http_biolib_result_uuid = "") {
+                return 400 "Missing biolib-result-uuid header";
+            }
+
+            if ($http_biolib_result_port = "") {
+                return 400 "Missing biolib-result-port header";
+            }
+
+            set $target_host "biolib-app-caller-proxy-$http_biolib_result_uuid";
+            proxy_pass http://$target_host:1080;
+            proxy_set_header biolib-result-uuid $http_biolib_result_uuid;
+            proxy_set_header biolib-result-port $http_biolib_result_port;
+            proxy_pass_request_headers on;
+        }
+    }
+}
+"""
+
+        nginx_config_bytes = nginx_config.encode()
+        tarfile_in_memory = io.BytesIO()
+        with tarfile.open(fileobj=tarfile_in_memory, mode='w:gz') as tar:
+            info = tarfile.TarInfo('/nginx.conf')
+            info.size = len(nginx_config_bytes)
+            tar.addfile(info, io.BytesIO(nginx_config_bytes))
+
+        tarfile_bytes = tarfile_in_memory.getvalue()
+        tarfile_in_memory.close()
+        docker.api.put_archive(self._container.id, '/etc/nginx', tarfile_bytes)

pybiolib-1.2.1117/biolib/compute_node/webserver/proxy_utils.py

@@ -0,0 +1,28 @@
+from docker.errors import ImageNotFound
+from docker.models.images import Image
+
+from biolib import utils
+from biolib.biolib_docker_client import BiolibDockerClient
+from biolib.biolib_logging import logger_no_user_data
+from biolib.typing_utils import cast
+
+
+def get_biolib_nginx_proxy_image() -> Image:
+    docker = BiolibDockerClient().get_docker_client()
+
+    if utils.IS_RUNNING_IN_CLOUD:
+        try:
+            logger_no_user_data.debug('Getting local Docker image for nginx proxy')
+            return cast(Image, docker.images.get('biolib-remote-host-proxy:latest'))
+        except ImageNotFound:
+            logger_no_user_data.debug(
+                'Local Docker image for nginx proxy not available. Falling back to public image...'
+            )
+
+    public_image_uri = 'public.ecr.aws/h5y4b3l1/biolib-remote-host-proxy:latest'
+    try:
+        logger_no_user_data.debug('Getting public Docker image for nginx proxy')
+        return cast(Image, docker.images.get(public_image_uri))
+    except ImageNotFound:
+        logger_no_user_data.debug('Pulling public Docker image for nginx proxy')
+        return cast(Image, docker.images.pull(public_image_uri))

{pybiolib-1.2.1111 → pybiolib-1.2.1117}/biolib/compute_node/webserver/webserver.py

@@ -17,6 +17,7 @@ from biolib.biolib_logging import TRACE, logger, logger_no_user_data
 from biolib.compute_node.cloud_utils.cloud_utils import CloudUtils
 from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
 from biolib.compute_node.webserver import webserver_utils
+from biolib.compute_node.webserver.compute_node_results_proxy import ComputeNodeResultsProxy
 from biolib.compute_node.webserver.gunicorn_flask_application import GunicornFlaskApplication
 from biolib.compute_node.webserver.webserver_utils import get_job_compute_state_or_404
 from biolib.typing_utils import Optional
@@ -152,6 +153,12 @@ def start_webserver(
             time.sleep(2)
 
         if utils.IS_RUNNING_IN_CLOUD:
+            try:
+                logger_no_user_data.debug('Stopping ComputeNodeResultsProxy')
+                ComputeNodeResultsProxy.stop_proxy()
+            except BaseException:
+                logger_no_user_data.exception('Failed to stop ComputeNodeResultsProxy')
+
             try:
                 logger_no_user_data.debug(f'Removing Docker network {BIOLIB_PROXY_NETWORK_NAME}')
                 docker_client = BiolibDockerClient.get_docker_client()
@@ -184,6 +191,12 @@ def start_webserver(
                 logger_no_user_data.exception(f'Failed to create Docker network {BIOLIB_PROXY_NETWORK_NAME}')
 
             if biolib_proxy_network:
+                try:
+                    logger_no_user_data.debug('Starting ComputeNodeResultsProxy')
+                    ComputeNodeResultsProxy.start_proxy(tls_pem_certificate_path, tls_pem_key_path)
+                except BaseException:
+                    logger_no_user_data.exception('Failed to start ComputeNodeResultsProxy')
+
                 CloudUtils.initialize()
                 webserver_utils.start_auto_shutdown_timer()
             else:

{pybiolib-1.2.1111 → pybiolib-1.2.1117}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pybiolib"
-version = "1.2.1111"
+version = "1.2.1117"
 description = "BioLib Python Client"
 readme = "PYPI_README.md"
 license = "MIT"