pybiolib 0.2.951__py3-none-any.whl → 1.2.1890__py3-none-any.whl

biolib/compute_node/remote_host_proxy.py

@@ -1,96 +1,140 @@
 import io
+import ipaddress
 import tarfile
-import subprocess
 import time
+from urllib.parse import urlparse
 
-from docker.models.containers import Container  # type: ignore
-from docker.errors import ImageNotFound  # type: ignore
-from docker.models.images import Image  # type: ignore
-from docker.models.networks import Network  # type: ignore
+from docker.models.containers import Container
+from docker.models.networks import Network
+from docker.types import EndpointConfig
 
 from biolib import utils
-from biolib.compute_node import enclave
-from biolib.compute_node.cloud_utils.enclave_parent_types import VsockProxyResponse
-from biolib.compute_node.cloud_utils import CloudUtils
-from biolib.typing_utils import Optional
-from biolib.biolib_api_client import RemoteHost
+from biolib._internal.utils import base64_encode_string
+from biolib.biolib_api_client import BiolibApiClient, RemoteHost
+from biolib.biolib_api_client.job_types import CreatedJobDict
 from biolib.biolib_docker_client import BiolibDockerClient
-from biolib.biolib_logging import logger
+from biolib.biolib_errors import BioLibError
+from biolib.biolib_logging import logger_no_user_data
+from biolib.compute_node.cloud_utils import CloudUtils
+from biolib.compute_node.utils import BIOLIB_PROXY_NETWORK_NAME
+from biolib.compute_node.webserver.proxy_utils import get_biolib_nginx_proxy_image
+from biolib.typing_utils import Dict, List, Optional
+
+
+def get_static_ip_from_network(network: Network, offset: int = 2) -> str:
+    ipam_config = network.attrs['IPAM']['Config']
+    if not ipam_config:
+        raise BioLibError(f'Network {network.name} has no IPAM configuration')
+
+    subnet_str = ipam_config[0]['Subnet']
+    subnet = ipaddress.ip_network(subnet_str, strict=False)
+
+    static_ip = str(subnet.network_address + offset)
+
+    return static_ip
 
 
 # Prepare for remote hosts with specified port
 class RemoteHostExtended(RemoteHost):
-    port: int
+    ports: List[int]
 
 
-class RemoteHostProxy:
-    _DOCKER_IMAGE_URI = 'nginx:1.21.1-alpine'
-    _TRAFFIC_FORWARDER_PORT_OFFSET = 10000  # Port offset relative to port of a VSOCK proxy
+class RemoteHostMapping:
+    def __init__(self, hostname: str, ports: List[int], network: Network, static_ip: str):
+        self.hostname = hostname
+        self.ports = ports
+        self.network = network
+        self.static_ip = static_ip
+
 
+class RemoteHostProxy:
     def __init__(
-            self,
-            remote_host: RemoteHost,
-            public_network: Network,
-            internal_network: Optional[Network],
-            job_id: Optional[str]
+        self,
+        remote_host_mappings: List[RemoteHostMapping],
+        job: CreatedJobDict,
+        app_caller_network: Optional[Network] = None,
     ):
-        # Default to port 443 for now until backend serves remote_hosts with port specified
-        self._remote_host: RemoteHostExtended = RemoteHostExtended(hostname=remote_host['hostname'], port=443)
-        self._public_network: Network = public_network
-        self._internal_network: Optional[Network] = internal_network
+        self._remote_host_mappings = remote_host_mappings
+        self._app_caller_network = app_caller_network
+        self.is_app_caller_proxy = app_caller_network is not None
 
-        if job_id:
-            self._name = f"biolib-remote-host-proxy-{job_id}-{self.hostname}"
-        else:
-            if not utils.BIOLIB_IS_RUNNING_IN_ENCLAVE:
-                raise Exception('RemoteHostProxy missing argument "job_id"')
-            self._name = f"biolib-enclave-remote-host-proxy-{self.hostname}"
+        if not job:
+            raise Exception('RemoteHostProxy missing argument "job"')
 
+        self._job = job
+        suffix = '-AppCallerProxy' if app_caller_network else ''
+        self._name = f'biolib-remote-host-proxy-{self._job_uuid}{suffix}'
         self._container: Optional[Container] = None
-        self._enclave_traffic_forwarder_process: Optional[subprocess.Popen] = None
-        self._enclave_vsock_proxy: Optional[VsockProxyResponse] = None
+        self._docker = BiolibDockerClient().get_docker_client()
 
     @property
-    def hostname(self) -> str:
-        return self._remote_host['hostname']
+    def _job_uuid(self) -> str:
+        return self._job['uuid']
+
+    def get_hostname_to_ip_mapping(self) -> Dict[str, str]:
+        return {mapping.hostname: mapping.static_ip for mapping in self._remote_host_mappings}
+
+    def get_remote_host_networks(self) -> List[Network]:
+        networks = [mapping.network for mapping in self._remote_host_mappings]
+        return networks
 
     def get_ip_address_on_network(self, network: Network) -> str:
         if not self._container:
-            raise Exception('RemoteHostProxy not yet started')
+            raise BioLibError('RemoteHostProxy not yet started')
 
         container_networks = self._container.attrs['NetworkSettings']['Networks']
         if network.name in container_networks:
             ip_address: str = container_networks[network.name]['IPAddress']
+            if not ip_address:
+                raise BioLibError(f'No IP address found for network {network.name}')
             return ip_address
 
-        raise Exception(f'RemoteHostProxy not connected to network {network.name}')
+        raise BioLibError(f'RemoteHostProxy not connected to network {network.name}')
 
     def start(self) -> None:
-        # TODO: Implement nice error handling in this method
+        docker = BiolibDockerClient.get_docker_client()
 
-        upstream_server_name = self._remote_host['hostname']
-        upstream_server_port = self._remote_host['port']
+        networking_config: Optional[Dict[str, EndpointConfig]] = (
+            None
+            if not self.is_app_caller_proxy
+            else {
+                BIOLIB_PROXY_NETWORK_NAME: docker.api.create_endpoint_config(
+                    aliases=[f'biolib-app-caller-proxy-{self._job_uuid}']
+                )
+            }
+        )
 
-        if utils.BIOLIB_IS_RUNNING_IN_ENCLAVE:
-            # In an enclave the flow is: application -> remote host proxy -> traffic forwarder -> VSOCK proxy -> remote
-            upstream_server_name = self._public_network.attrs['IPAM']['Config'][0]['Gateway']
-            upstream_server_port = self._start_vsock_proxy_and_traffic_forwarder_and_return_local_port()
+        for index in range(3):
+            logger_no_user_data.debug(f'Attempt {index} at creating RemoteHostProxy container "{self._name}"...')
+            try:
+                self._container = docker.containers.create(
+                    detach=True,
+                    image=get_biolib_nginx_proxy_image(),
+                    name=self._name,
+                    network=BIOLIB_PROXY_NETWORK_NAME,
+                    networking_config=networking_config,
+                )
+                break
+            except Exception as error:
+                logger_no_user_data.exception(f'Failed to create container "{self._name}" hit error: {error}')
 
-        docker = BiolibDockerClient.get_docker_client()
-        self._container = docker.containers.create(
-            detach=True,
-            image=self._get_nginx_docker_image(),
-            name=self._name,
-            network=self._public_network.name,
-        )
+            logger_no_user_data.debug('Sleeping before re-trying container creation...')
+            time.sleep(3)
 
-        self._write_nginx_config_to_container(
-            upstream_server_name,
-            upstream_server_port,
-        )
+        if not self._container or not self._container.id:
+            raise BioLibError(f'Exceeded re-try limit for creating container {self._name}')
+
+        for mapping in self._remote_host_mappings:
+            mapping.network.connect(self._container.id, ipv4_address=mapping.static_ip)
+            logger_no_user_data.debug(
+                f'Connected proxy to network {mapping.network.name} with static IP {mapping.static_ip}'
+            )
 
-        if self._internal_network:
-            self._internal_network.connect(self._container.id)
+        if self._app_caller_network:
+            self._app_caller_network.connect(self._container.id)
+            logger_no_user_data.debug(f'Connected app caller proxy to network {self._app_caller_network.name}')
+
+        self._write_nginx_config_to_container()
 
         self._container.start()
 
@@ -99,7 +143,8 @@ class RemoteHostProxy:
             time.sleep(0.5 * retry_count)
             # Use the container logs as a health check.
             # By using logs instead of a http endpoint on the NGINX we avoid publishing a port of container to the host
-            if b'start worker process ' in self._container.logs():
+            container_logs = self._container.logs()
+            if b'ready for start up\n' in container_logs or b'start worker process ' in container_logs:
                 proxy_is_ready = True
                 break
 
@@ -115,58 +160,315 @@ class RemoteHostProxy:
         if self._container:
             self._container.remove(force=True)
 
-        if self._enclave_traffic_forwarder_process:
-            self._enclave_traffic_forwarder_process.terminate()
-
-        if self._enclave_vsock_proxy:
-            CloudUtils.enclave.stop_vsock_proxy(self._enclave_vsock_proxy['id'])
+    def _write_nginx_config_to_container(self) -> None:
+        if not self._container:
+            raise Exception('RemoteHostProxy container not defined when attempting to write NGINX config')
 
-    def _get_nginx_docker_image(self) -> Image:
         docker = BiolibDockerClient.get_docker_client()
-        try:
-            return docker.images.get(self._DOCKER_IMAGE_URI)
-        except ImageNotFound:
-            logger.debug('Pulling remote host docker image...')
-            return docker.images.pull(self._DOCKER_IMAGE_URI)
+        upstream_hostname = urlparse(BiolibApiClient.get().base_url).hostname
+        if self.is_app_caller_proxy:
+            if not utils.IS_RUNNING_IN_CLOUD:
+                raise BioLibError('Calling apps inside apps is not supported in local compute environment')
+
+            logger_no_user_data.debug(f'Job "{self._job_uuid}" writing config for and starting App Caller Proxy...')
+            config = CloudUtils.get_webserver_config()
+            compute_node_uuid = config['compute_node_info']['public_id']
+            compute_node_auth_token = config['compute_node_info']['auth_token']
+
+            # TODO: Get access_token from new API class instead
+            access_token = BiolibApiClient.get().access_token
+            bearer_token = f'Bearer {access_token}' if access_token else ''
+
+            user_uuid = self._job.get('user_id')
+            if user_uuid:
+                biolib_index_basic_auth = (
+                    f'biolib_user|{user_uuid.replace("-", "_")}:cloud-{compute_node_auth_token},{self._job_uuid}'
+                )
+                biolib_index_auth_header_value = f'Basic {base64_encode_string(biolib_index_basic_auth)}'
+                logger_no_user_data.debug(f'Job "{self._job_uuid}" using biolib_user auth for biolib-index')
+            else:
+                biolib_index_auth_header_value = ''
+                logger_no_user_data.debug(f'Job "{self._job_uuid}" has no biolib-index auth configured')
+
+            nginx_config = f"""
+events {{
+  worker_connections  1024;
+}}
 
-    def _start_vsock_proxy_and_traffic_forwarder_and_return_local_port(self) -> int:
-        # TODO: Implement nice error handling in this method
+http {{
+    client_max_body_size 0;
+    map $request_method $bearer_token_on_post {{
+        POST       "{bearer_token}";
+        default    "";
+    }}
 
-        if not utils.BIOLIB_IS_RUNNING_IN_ENCLAVE:
-            raise Exception('VSOCK proxy and traffic forwarder should only be started in enclave')
+    map $request_method $bearer_token_on_get {{
+        GET        "{bearer_token}";
+        default    "";
+    }}
 
-        logger.debug(f"Requesting vsock proxy for hostname {self.hostname}")
-        self._enclave_vsock_proxy = CloudUtils.enclave.start_vsock_proxy(self._remote_host)
+    map $request_method $bearer_token_on_patch_and_get {{
+        PATCH      "{bearer_token}";
+        GET        "{bearer_token}";
+        default    "";
+    }}
 
-        traffic_forwarder_local_port = self._enclave_vsock_proxy['port'] + self._TRAFFIC_FORWARDER_PORT_OFFSET
-        logger.debug(
-            f"Starting traffic forwarder on local port {traffic_forwarder_local_port} to "
-            f"VSOCK port {self._enclave_vsock_proxy['port']} for hostname {self.hostname}"
-        )
-        self._enclave_traffic_forwarder_process = subprocess.Popen([
-            'biolib_traffic_forwarder',
-            str(traffic_forwarder_local_port),
-            str(enclave.PARENT_CID),
-            str(self._enclave_vsock_proxy['port']),
-        ])
-        return traffic_forwarder_local_port
-
-    def _write_nginx_config_to_container(self, upstream_server_name: str, upstream_server_port: int) -> None:
-        if not self._container:
-            raise Exception('RemoteHostProxy container not defined when attempting to write NGINX config')
+    server {{
+        listen       80;
+        resolver 127.0.0.11 ipv6=off valid=30s;
+        set $upstream_hostname {upstream_hostname};
+
+        location ~* "^/api/jobs/cloud/(?<job_id>[a-z0-9-]{{36}})/status/$" {{
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$job_id/status/;
+            proxy_set_header         authorization $bearer_token_on_get;
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/cloud/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/cloud/$1;
+            proxy_set_header         authorization $bearer_token_on_post;
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/start_upload/$" {{
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/start_upload/;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/presigned_upload_url/$" {{
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/presigned_upload_url/$is_args$args;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/storage/input/complete_upload/$" {{
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/storage/input/complete_upload/;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/main_result/$" {{
+            proxy_pass                  https://$upstream_hostname/api/jobs/$job_id/main_result/;
+            proxy_set_header            authorization "";
+            proxy_set_header            cookie "";
+            proxy_pass_request_headers  on;
+            proxy_ssl_server_name       on;
+        }}
+
+        location ~* "^/api/jobs/(?<job_id>[a-z0-9-]{{36}})/$" {{
+            proxy_pass               https://$upstream_hostname/api/jobs/$job_id/;
+            proxy_set_header         authorization $bearer_token_on_patch_and_get;
+            proxy_set_header         caller-job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/create_job_with_data/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/create_job_with_data/$1;
+            proxy_set_header         authorization $bearer_token_on_post;
+            proxy_set_header         caller-job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/jobs/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/$1;
+            proxy_set_header         authorization $bearer_token_on_post;
+            proxy_set_header         caller-job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~ "^/api/jobs/{self._job_uuid}/notes/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/jobs/{self._job_uuid}/notes/$1;
+            proxy_set_header         authorization "";
+            proxy_set_header         job-auth-token "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         compute-node-uuid "{compute_node_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~ "^/api/auth/oauth-token-exchange/$" {{
+            # Note: Using $1 here as URI part from regex must be used for proxy_pass
+            proxy_pass               https://$upstream_hostname/api/auth/oauth-token-exchange/$1;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/lfs/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/app/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/resource/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/resources/data-records/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/resources/versions/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/proxy/files/data-record-versions/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/proxy/index/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "{biolib_index_auth_header_value}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location ~* "^/api/accounts/(?<account_id>[a-z0-9-]{{36}})/metrics/jobs/$" {{
+            proxy_pass               https://$upstream_hostname/api/accounts/$account_id/metrics/jobs/$is_args$args;
+            proxy_set_header         authorization "";
+            proxy_set_header         compute-node-auth-token "{compute_node_auth_token}";
+            proxy_set_header         job-uuid "{self._job_uuid}";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /api/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /proxy/storage/job-storage/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /proxy/storage/lfs/versions/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location /proxy/cloud/ {{
+            proxy_pass               https://$upstream_hostname$request_uri;
+            proxy_set_header         authorization "";
+            proxy_set_header         cookie "";
+            proxy_ssl_server_name    on;
+        }}
+
+        location / {{
+            return 404 "Not found";
+        }}
+    }}
 
-        docker = BiolibDockerClient.get_docker_client()
-        nginx_config = f'''
-events {{}}
-error_log /dev/stdout info;
-stream {{
     server {{
-        listen          {self._remote_host['port']};
-        proxy_pass      {upstream_server_name}:{upstream_server_port};
+        listen       1080;
+        resolver 127.0.0.11 ipv6=off valid=30s;
+
+        if ($http_biolib_result_uuid != "{self._job_uuid}") {{
+            return 403 "Invalid or missing biolib-result-uuid header";
+        }}
+
+        if ($http_biolib_result_port = "") {{
+            return 400 "Missing biolib-result-port header";
+        }}
+
+        location / {{
+            proxy_pass               http://main:$http_biolib_result_port$request_uri;
+            proxy_set_header         Host $http_host;
+            proxy_set_header         biolib-result-uuid "";
+            proxy_set_header         biolib-result-port "";
+            proxy_pass_request_headers on;
+        }}
     }}
 }}
+"""
+        else:
+            port_to_mappings: Dict[int, List[RemoteHostMapping]] = {}
+            for mapping in self._remote_host_mappings:
+                for port in mapping.ports:
+                    if port not in port_to_mappings:
+                        port_to_mappings[port] = []
+                    port_to_mappings[port].append(mapping)
+
+            nginx_config = """
+events {}
+error_log /dev/stdout info;
+stream {
+    resolver 127.0.0.11 valid=30s;"""
+
+            for port, mappings in port_to_mappings.items():
+                nginx_config += f"""
+    map $server_addr $backend_{port} {{"""
+                for mapping in mappings:
+                    nginx_config += f'\n        {mapping.static_ip} {mapping.hostname}:{port};'
+
+                nginx_config += f"""
+    }}
+    server {{
+        listen 0.0.0.0:{port};
+        proxy_pass $backend_{port};
+    }}
+    server {{
+        listen 0.0.0.0:{port} udp;
+        proxy_pass $backend_{port};
+    }}"""
+
+            nginx_config += """
+}
+"""
 
-'''
         nginx_config_bytes = nginx_config.encode()
         tarfile_in_memory = io.BytesIO()
         with tarfile.open(fileobj=tarfile_in_memory, mode='w:gz') as tar:

biolib/compute_node/utils.py

@@ -1,36 +1,41 @@
-from enum import Enum
 import random
 import string
+from enum import Enum
 
 from biolib.biolib_logging import logger
 
+BIOLIB_PROXY_NETWORK_NAME = 'biolib-proxy-network'
+
 
 def get_package_type(package):
     package_type = int.from_bytes(package[1:2], 'big')
     if package_type == 1:
         return 'ModuleInput'
     elif package_type == 2:
-        return 'ModuleOutput'
-    elif package_type == 3:  # Not used in the compute node
-        return 'ModuleSource'
+        return 'ModuleOutput'  # Note: This package is deprecated
+    elif package_type == 3:
+        return 'ModuleSource'  # Note: This package is deprecated
     elif package_type == 4:
-        return 'AttestationDocument'
+        return 'AttestationDocument'  # Note: This package is deprecated
     elif package_type == 5:
         return 'SavedJob'
     elif package_type == 6:
-        return 'RsaEncryptedAesPackage'
+        return 'RsaEncryptedAesPackage'  # Note: This package is deprecated
     elif package_type == 7:
-        return 'AesEncryptedPackage'
+        return 'AesEncryptedPackage'  # Note: This package is deprecated
     elif package_type == 8:
         return 'SystemStatusUpdate'
     elif package_type == 9:
         return 'SystemException'
+    elif package_type == 10:
+        return 'StdoutAndStderr'
 
     else:
-        raise Exception(f"Unexpected package type {package_type}")
+        raise Exception(f'Unexpected package type {package_type}')
 
 
 class SystemExceptionCodes(Enum):
+    COMPLETED_SUCCESSFULLY = 0
     FAILED_TO_INIT_COMPUTE_PROCESS_VARIABLES = 1
     FAILED_TO_CONNECT_TO_WORKER_THREAD_SOCKET = 2
     FAILED_TO_START_SENDER_THREAD_OR_RECEIVER_THREAD = 3
@@ -55,13 +60,22 @@ class SystemExceptionCodes(Enum):
     FAILED_TO_RETRIEVE_AND_MAP_OUTPUT_FILES = 22
     FAILED_TO_SERIALIZE_AND_SEND_MODULE_OUTPUT = 23
     FAILED_TO_DESERIALIZE_SAVED_JOB = 24
-    UNKOWN_COMPUTE_PROCESS_ERROR = 25
+    UNKNOWN_COMPUTE_PROCESS_ERROR = 25
     FAILED_TO_INITIALIZE_WORKER_THREAD = 26
     FAILED_TO_HANDLE_PACKAGE_IN_WORKER_THREAD = 27
     EXCEEDED_MAX_JOB_RUNTIME = 28
+    OUT_OF_MEMORY = 29
+    CANCELLED_BY_USER = 30
+    COMMAND_OVERRIDE_NOT_ALLOWED = 31
+    FAILED_TO_ALLOCATE_JOB_TO_COMPUTE_NODE = 32
+    NO_MODULES_FOUND_ON_JOB = 33
+    FAILED_TO_INITIALIZE_DOCKER_EXECUTOR = 34
+    COMPUTE_NODE_SHUTDOWN = 35
+    COMPUTE_NODE_SHUTDOWN_DUE_TO_HEALTH_CHECK_FAILURE = 36
 
 
 SystemExceptionCodeMap = {
+    0: 'Job completed successfully',
     1: 'Failed to init compute process variables',
     2: 'Failed to connect to worker thread socket',
     3: 'Failed to start sender or receiver thread',
@@ -90,6 +104,14 @@ SystemExceptionCodeMap = {
     26: 'Failed to initialize worker thread',
     27: 'Failed to handle package in worker thread',
     28: 'Job exceeded max run time',
+    29: 'Container ran out of memory',
+    30: 'Job was cancelled by the user',
+    31: 'The application does not allow the arguments to override the command',
+    32: 'Failed to allocate job to compute node',
+    33: 'No modules found on job',
+    34: 'Failed to initialize docker executor',
+    35: 'Compute node shutdown',
+    36: 'Compute node shutdown due to health check failure',
 }