matlab-proxy 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

matlab_proxy_manager/storage/server.py

@@ -0,0 +1,144 @@
+# Copyright 2024 The MathWorks, Inc.
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Optional
+
+from matlab_proxy_manager.utils import helpers, logger
+
+log = logger.get()
+
+
+@dataclass
+class ServerProcess:
+    """
+    Represents a MATLAB server process with various attributes and methods
+    to manage its lifecycle.
+    """
+
+    server_url: Optional[str] = None
+    mwi_base_url: Optional[str] = None
+    headers: Optional[dict] = None
+    errors: Optional[list] = None
+    pid: Optional[int] = None
+    parent_pid: Optional[int] = None
+    absolute_url: Optional[str] = field(default=None)
+    id: Optional[str] = None
+    type: Optional[str] = None
+    mpm_auth_token: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        if not self.absolute_url:
+            self.absolute_url = f"{self.server_url}{self.mwi_base_url}"
+
+    def __str__(self) -> str:
+        """
+        Returns a string representation of the ServerProcess instance.
+        """
+        return json.dumps(asdict(self))
+
+    def as_dict(self) -> dict:
+        """
+        Returns a dict representation of the ServerProcess instance.
+        """
+        return asdict(self)
+
+    @staticmethod
+    def instantiate_from_string(data: str) -> "ServerProcess":
+        """
+        Instantiates a ServerProcess object from a JSON string.
+
+        Args:
+            data (str): The JSON string representing a ServerProcess.
+
+        Returns:
+            ServerProcess: An instance of ServerProcess.
+
+        Raises:
+            ValueError: If the JSON string cannot be parsed or is missing required fields.
+        """
+        try:
+            full_dict = json.loads(data)
+            key = list(full_dict.keys())[0]
+            server = full_dict[key]
+            server_process = ServerProcess(
+                server_url=server["server_url"],
+                mwi_base_url=server["mwi_base_url"],
+                headers=server["headers"],
+                errors=server["errors"],
+                pid=server["pid"],
+                parent_pid=server["parent_pid"],
+                absolute_url=server["absolute_url"],
+                id=server["id"],
+                type=server["type"],
+                mpm_auth_token=server["mpm_auth_token"],
+            )
+            return server_process
+        except (json.JSONDecodeError, KeyError) as ex:
+            log.debug("Failed to instantiate server from %s: %s", data, ex)
+            raise ValueError(
+                "Invalid JSON string for ServerProcess instantiation"
+            ) from ex
+
+    def shutdown(self):
+        """
+        Shuts down the MATLAB proxy server by calling the shutdown_integration endpoint.
+        """
+        log.debug("Shutting down matlab proxy")
+        backend_server = self.absolute_url
+        url = f"{backend_server}/shutdown_integration"
+        try:
+            response = helpers.requests_retry_session(retries=1).delete(
+                url=url, headers=self.headers
+            )
+            shutdown_resp = response.json()
+            log.debug("Response from shutdown: %s", response.json())
+            return shutdown_resp
+        except Exception as e:
+            log.debug("Exception while shutting down matlab proxy: %s", e)
+            return None
+
+    def is_server_alive(self) -> bool:
+        """
+        Checks if the server process is alive and ready.
+
+        Returns:
+            bool: True if the server process is alive and ready, False otherwise.
+        """
+        return helpers.does_process_exist(self.pid) and helpers.is_server_ready(
+            self.absolute_url, retries=0
+        )
+
+    @staticmethod
+    def find_existing_server(data_dir, key: str) -> Optional["ServerProcess"]:
+        """
+        Finds an existing server process by reading the server configuration from a file.
+
+        Args:
+            data_dir (str): The directory where server configuration files are stored.
+            key (str): The key corresponding to the specific server configuration.
+
+        Returns:
+            Optional[ServerProcess]: An instance of ServerProcess if a valid configuration is found,
+            otherwise None.
+        """
+        key_dir = Path(data_dir, key)
+        server_process = None
+
+        # Return early if the directory is not found
+        if not key_dir.is_dir():
+            return server_process
+
+        files = list(key_dir.iterdir())
+        if not files:
+            return server_process
+
+        try:
+            with open(files[0], "r", encoding="utf-8") as file:
+                data = file.read().strip()
+                if data:
+                    server_process = ServerProcess.instantiate_from_string(data)
+        except (OSError, ValueError) as ex:
+            log.debug("Exception while checking for existing server: %s", ex)
+
+        return server_process

matlab_proxy_manager/utils/__init__.py

@@ -0,0 +1 @@
+# Copyright 2024 The MathWorks, Inc.

matlab_proxy_manager/utils/auth.py

@@ -0,0 +1,77 @@
+# Copyright 2024 The MathWorks, Inc.
+from hmac import compare_digest
+
+from aiohttp import web
+
+from matlab_proxy_manager.utils import logger
+from matlab_proxy_manager.utils.constants import HEADER_MWI_MPM_AUTH_TOKEN
+
+log = logger.get()
+
+
+async def authenticate_request(request):
+    """Authenticates incoming request by verifying whether the expected token is in the request.
+
+    The MWI-MPM-AUTH-TOKEN must be present in the request's headers.
+
+    Returns True/False based on whether the request is authenticated.
+    """
+
+    log.debug("<======== Authenticate request: %s", request)
+    return await _is_valid_token_in_headers(request)
+
+
+def authenticate_access_decorator(endpoint):
+    """This decorator verifies that the request to an endpoint exposed by matlab-proxy-manager
+    contains the correct mpm auth token before servicing an endpoint."""
+
+    async def protect_endpoint(request):
+        """Passes request to the endpoint after validating the token
+
+        Args:
+            request (HTTPRequest) : Web Request to endpoint
+
+        Raises:
+            web.HTTPForbidden: Thrown when validation of token fails
+        """
+        if await authenticate_request(request):
+            # request is authentic, proceed to execute the endpoint
+            return await endpoint(request)
+        raise web.HTTPForbidden(reason="Unauthorized access!")
+
+    return protect_endpoint
+
+
+async def _is_valid_token_in_headers(request):
+    """Checks the request headers for mpm auth token
+
+    Args:
+        request (HTTPRequest) : Used to access app settings
+
+    Returns:
+        Boolean : True if valid token is found, else False
+    """
+    headers = request.headers
+    if HEADER_MWI_MPM_AUTH_TOKEN in headers:
+        return await _is_valid_token(headers[HEADER_MWI_MPM_AUTH_TOKEN], request)
+
+    log.debug("Header: %s not found in request headers", HEADER_MWI_MPM_AUTH_TOKEN)
+    return False
+
+
+async def _is_valid_token(token, request):
+    """Checks if token contains expected value.
+
+    Args:
+        token (str): Token string to validate
+        request (HTTPRequest) : Used to access app settings
+
+    Returns:
+        _type_: True if token is valid, false otherwise.
+    """
+    # Check if the token provided in the request matches the original token
+    # equivalent to a == b, but protects against timing attacks
+    saved_token = request.app["auth_token"]
+    is_valid = compare_digest(token, saved_token)
+    log.debug("Token validation %s ", "successful." if is_valid else "failed.")
+    return is_valid

matlab_proxy_manager/utils/constants.py

@@ -0,0 +1,5 @@
+# Copyright 2024 The MathWorks, Inc.
+
+MWI_BASE_URL_PREFIX = "/matlab/"
+HEADER_MWI_MPM_CONTEXT = "MWI-MPM-CONTEXT"
+HEADER_MWI_MPM_AUTH_TOKEN = "MWI-MPM-AUTH-TOKEN"

matlab_proxy_manager/utils/environment_variables.py

@@ -0,0 +1,46 @@
+# Copyright 2020-2024 The MathWorks, Inc.
+"""This file lists and exposes the environment variables which are used by proxy manager."""
+
+import os
+
+
+def _is_env_set_to_true(env_name: str) -> bool:
+    """Helper function that returns True if the environment variable specified is set to True.
+
+    Args:
+        env_name (str): Name of the environment variable to check the state for.
+
+    Returns:
+        bool: True if the environment variable's value matches(case-insensitive) the string "True"
+    """
+    return os.environ.get(env_name, "").lower() == "true"
+
+
+def get_env_name_logging_level():
+    """Specifies the logging level used by app's loggers"""
+    return "MWI_MPM_LOG_LEVEL"
+
+
+def get_env_name_enable_web_logging():
+    """Enable the logging of asyncio web traffic by setting to true"""
+    return "MWI_MPM_ENABLE_WEB_LOGGING"
+
+
+def get_env_name_mwi_mpm_auth_token():
+    """Authentication environment variable for authenticating with proxy manager"""
+    return "MWI_MPM_AUTH_TOKEN"
+
+
+def get_env_name_mwi_mpm_port():
+    """Used to specify the port on which to start proxy manager"""
+    return "MWI_MPM_PORT"
+
+
+def get_env_name_mwi_mpm_parent_pid():
+    """Used to specify the parent pid for the proxy manager process"""
+    return "MWI_MPM_PARENT_PID"
+
+
+def is_web_logging_enabled():
+    """Returns true if the web logging is required to be enabled"""
+    return _is_env_set_to_true(get_env_name_enable_web_logging())

matlab_proxy_manager/utils/helpers.py

@@ -0,0 +1,286 @@
+# Copyright 2024 The MathWorks, Inc.
+import os
+import socket
+import time
+from pathlib import Path
+from typing import Dict
+
+import psutil
+import requests
+from aiohttp import web
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from matlab_proxy import settings
+from matlab_proxy_manager.storage.file_repository import FileRepository
+from matlab_proxy_manager.utils import logger
+
+log = logger.get()
+
+
+def is_server_ready(url: str, retries: int = 2, backoff_factor=None) -> bool:
+    """
+    Check if the server at the given URL is ready.
+
+    Args:
+        url (str): The URL of the server.
+
+    Returns:
+        bool: True if the server is ready, False otherwise.
+    """
+    try:
+        matlab_proxy_index_page_identifier = "MWI_MATLAB_PROXY_IDENTIFIER"
+        resp = requests_retry_session(
+            retries=retries, backoff_factor=backoff_factor
+        ).get(f"{url}", verify=False)
+        log.debug("Response status code from server readiness: %s", resp.status_code)
+        return (
+            resp.status_code == requests.codes.OK
+            and matlab_proxy_index_page_identifier in resp.text
+        )
+    except Exception as e:
+        log.debug("Couldn't reach the server with error: %s", e)
+        return False
+
+
+def requests_retry_session(
+    retries=3, backoff_factor=0.1, session=None
+) -> requests.Session:
+    """
+    Create a requests session with retry logic.
+
+    Args:
+        retries (int): The number of retries.
+        backoff_factor (float): The backoff factor for retries.
+        session (requests.Session, optional): An existing requests session.
+
+    Returns:
+        requests.Session: The requests session with retry logic.
+    """
+    session = session or requests.session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+        backoff_factor=backoff_factor,
+        allowed_methods=frozenset(["DELETE", "GET", "POST"]),
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    return session
+
+
+def does_process_exist(pid: int) -> bool:
+    """
+    Checks if the parent process is alive.
+
+    Returns:
+        bool: True if the parent process is alive, False otherwise.
+    """
+    parent_status = pid is not None and psutil.pid_exists(int(pid))
+    log.debug("Parent liveness check returned: %s", parent_status)
+    return parent_status
+
+
+def convert_mwi_env_vars_to_header_format(
+    env_vars: Dict[str, str], prefix: str
+) -> Dict[str, str]:
+    """
+    Parse and transform environment variables with a specific prefix.
+
+    Args:
+        env_vars (dict): The environment variables.
+        prefix (str): The prefix to filter the environment variables.
+
+    Returns:
+        dict: The transformed environment variables.
+    """
+    return {
+        key.replace("_", "-"): value
+        for key, value in env_vars.items()
+        if key.startswith(prefix)
+    }
+
+
+def create_and_get_proxy_manager_data_dir() -> Path:
+    """
+    Create and get the proxy manager data directory.
+
+    Returns:
+        Path: The path to the proxy manager data directory.
+    """
+
+    config_dir = settings.get_mwi_config_folder(dev=False)
+    data_dir = Path(config_dir, "proxy_manager")
+    Path.mkdir(data_dir, parents=True, exist_ok=True)
+    return data_dir
+
+
+async def delete_dangling_servers(app: web.Application) -> None:
+    """
+    Delete dangling matlab proxy servers that are no longer alive.
+
+    Args:
+        app (web.Application): aiohttp web application
+    """
+    is_delete_successful = _are_orphaned_servers_deleted(None)
+    log.debug("Deleted dangling matlab proxy servers: %s", is_delete_successful)
+
+
+def _are_orphaned_servers_deleted(predicate: str) -> bool:
+    """
+    Get all the files under the proxy manager directory, check the status of the servers,
+    and delete orphaned servers and their corresponding files.
+
+    - Checks if the parent PID of each server is still alive. If not, sends a SIGKILL
+    to the server and deletes the corresponding file.
+    - Checks if the servers in those files are still alive by sending GET requests to
+    their absolute URLs. If not, deletes the corresponding file.
+
+    Returns:
+        bool: True if any server was deleted, False otherwise.
+    """
+    data_dir = create_and_get_proxy_manager_data_dir()
+    storage = FileRepository(data_dir)
+    servers: dict = storage.get_all()
+
+    def _matches_predicate(filename: str) -> bool:
+        return filename.split("_")[0] == predicate
+
+    # Checks only a subset of servers (that matches the parent_pid of the caller)
+    # to reduce the MATLAB proxy startup time
+    if predicate:
+        servers = {
+            filename: server
+            for filename, server in servers.items()
+            if _matches_predicate(filename)
+        }
+        if not servers:
+            log.debug("Parent pid not matched, nothing to cleanup")
+            return True
+
+    return _delete_server_and_file(storage, servers)
+
+
+def _delete_server_and_file(storage, servers):
+    is_server_deleted = False
+    for filename, server in servers.items():
+        if not server.is_server_alive():
+            log.debug("Server is not alive, cleaning up files")
+            try:
+                storage.delete(os.path.basename(filename))
+            except Exception as ex:
+                log.debug("Failed to delete file: %s", ex)
+            is_server_deleted = True
+        elif not does_process_exist(server.parent_pid):
+            log.debug("Server's parent is gone, shutting down matlab proxy")
+            try:
+                server.shutdown()
+            except Exception as ex:
+                log.debug("Failed to shutdown the matlab proxy server: %s", ex)
+            finally:
+                # Ensures files are cleaned up even if shutdown fails
+                storage.delete(os.path.basename(filename))
+                is_server_deleted = True
+
+    return is_server_deleted
+
+
+def poll_for_server_deletion() -> None:
+    """
+    Poll for server deletion for a specified timeout period.
+
+    This function continuously checks if orphaned servers are deleted within a
+    specified timeout period. If servers are deleted, it breaks out of the loop.
+
+    Logs the status of server deletion attempts.
+    """
+    timeout_in_seconds: int = 2
+    log.info("Interrupt/termination signal caught, cleaning up resources")
+    start_time = time.time()
+
+    while time.time() - start_time < timeout_in_seconds:
+        is_server_deleted = _are_orphaned_servers_deleted(None)
+        if is_server_deleted:
+            log.debug("Servers deleted, breaking out of loop")
+            break
+        log.debug("Servers not deleted, waiting")
+        # Sleep for a short interval before polling again
+        time.sleep(0.5)
+
+
+def find_free_port() -> str:
+    """
+    Find a free port on the system.
+
+    This function creates a socket, binds it to an available port, retrieves
+    the port number, and then closes the socket.
+
+    Returns:
+        str: The free port number as a string.
+    """
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("", 0))
+    port = str(s.getsockname()[1])
+    s.close()
+    return port
+
+
+def pre_load_from_state_file(data_dir: str) -> Dict[str, str]:
+    """
+    Pre-load server states from the state files in the specified data directory.
+
+    Args:
+        data_dir (Path): The directory containing the state files.
+
+    Returns:
+        Dict[str, Dict]: A dictionary with server IDs as keys and server states as values.
+    """
+    storage = FileRepository(data_dir)
+    servers = storage.get_all()
+    return {server.id: server.as_dict() for server in servers.values()}
+
+
+def is_only_reference(file_path: str) -> bool:
+    """
+    Check if the specified file is the only file in its directory.
+
+    Args:
+        file_path (str): The path to the file.
+
+    Returns:
+        bool: True if the file is the only file in its directory, False otherwise.
+    """
+    parent_dir = Path(file_path).parent.absolute()
+    files = os.listdir(parent_dir)
+    return len(files) == 1 and files[0] == os.path.basename(file_path)
+
+
+def create_state_file(data_dir, server_process, filename: str):
+    """
+    Create a state file in the specified data directory.
+
+    This function creates a state file for the given server process in the specified
+    data directory. It logs the process and handles any exceptions that might occur.
+
+    Args:
+        data_dir: The directory where the state file will be created.
+        server_process (ServerProcess): The server process for which the state file is created.
+        filename (str): The name of the state file to be created.
+
+    Raises:
+        IOError: If there is an error creating the state file or adding the server
+        process to the repository.
+    """
+    try:
+        storage = FileRepository(data_dir)
+        storage.add(server=server_process, filename=filename)
+        log.debug("State file %s created in %s", filename, data_dir)
+    except Exception as e:
+        log.error(
+            "Failed to create state file %s in %s, error: %s", filename, data_dir, e
+        )
+        raise IOError(
+            f"Failed to create state file {filename} in {data_dir}, error: {e}"
+        ) from e

matlab_proxy_manager/utils/logger.py

@@ -0,0 +1,73 @@
+# Copyright 2024 The MathWorks, Inc.
+# Helper functions to access & control the logging behavior of the app
+
+import logging
+import os
+
+
+# TODO: Consolidate all logging setup into a common module (MATLABProxy, MATLABKernel, MATLABProxyManager)
+def get(init=False):
+    """Get the logger used by this application.
+        Set init=True to initialize the logger
+    Returns:
+        Logger: The logger used by this application.
+    """
+    if init is True:
+        return __set_logging_configuration()
+
+    return __get_mw_logger()
+
+
+def __get_mw_logger_name():
+    """Name of logger used by the app
+
+    Returns:
+        String: The name of the Logger.
+    """
+    return "MATLABProxyManager"
+
+
+def __get_mw_logger():
+    """Returns logger for use in this app.
+
+    Returns:
+        Logger: A logger object
+    """
+    return logging.getLogger(__get_mw_logger_name())
+
+
+def __set_logging_configuration():
+    """Sets the logging environment for the app
+
+    Returns:
+        Logger: Logger object with the set configuration.
+    """
+    # query for user specified environment variables
+    log_level = os.getenv(__get_env_name_logging_level(), __get_default_log_level())
+
+    ## Set logging object
+    logger = __get_mw_logger()
+
+    # log_level is either set by environment or is the default value.
+    logger.info("Initializing logger with log_level: %s", log_level)
+    logger.setLevel(log_level)
+
+    # Allow other libraries used by this integration to
+    # also print their logs at the specified level
+    logging.basicConfig(level=log_level)
+
+    return logger
+
+
+def __get_env_name_logging_level():
+    """Specifies the logging level used by app's loggers"""
+    return "MWI_MPM_LOG_LEVEL"
+
+
+def __get_default_log_level():
+    """The default logging level used by this application.
+
+    Returns:
+        String: The default logging level
+    """
+    return "INFO"

matlab_proxy_manager/web/__init__.py

@@ -0,0 +1 @@
+# Copyright 2024 The MathWorks, Inc.