splight-runner 1.0.0__py3-none-any.whl

splight_runner/__init__.py

@@ -0,0 +1,3 @@
+from splight_runner.runner import runner
+
+__all__ = [runner]

splight_runner/api/component_reporter.py

@@ -0,0 +1,55 @@
+import requests
+
+from splight_runner.api.settings import settings
+
+
+class StatusReporter:
+    """Class responsible for updating the component status each time the
+    healthcheck method is called.
+    """
+
+    _SPL_PREFIX = "Splight"
+    _BASE_PATH = "v2/engine/component/components"
+
+    def __init__(
+        self, api_host: str, access_id: str, secret_key: str, component_id: str
+    ):
+        self._api_host = api_host.rstrip("/")
+        self._component_id = component_id
+        self._auth_header = {
+            "Authorization": f"{self._SPL_PREFIX} {access_id} {secret_key}"
+        }
+        self._prev_status: str = "Unknown"
+
+    def report_status(self, status: str) -> None:
+        """Updates the status of the component in the Splight Platform making
+        POST request to the API.
+
+        Parameters
+        ----------
+        status : str
+            The status of the component.
+            It can be "Running", "Stopped" or "Succeeded".
+        """
+        if self._prev_status == status:
+            return None
+        url = f"{self._api_host}/{self._BASE_PATH}"
+        url = f"{url}/{self._component_id}/update-status/"
+        response = requests.post(
+            url, headers=self._auth_header, data={"deployment_status": status}
+        )
+        try:
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as exc:
+            print("Unable to update component status")
+            print(exc)
+        self._prev_status = status
+        return None
+
+
+reporter = StatusReporter(
+    api_host=settings.splight_platform_api_host,
+    access_id=settings.access_id,
+    secret_key=settings.secret_key,
+    component_id=settings.process_id,
+)

splight_runner/api/logging_interceptor.py

@@ -0,0 +1,85 @@
+import traceback
+from logging import LogRecord
+
+from splight_runner.api.settings import settings
+from splight_runner.log_streamer.log_streamer import ComponentLogsStreamer
+from splight_runner.logging import log
+
+
+class ApplicationLogInterceptor:
+    """Class responsible for intercept logs records from the logging module
+    and append them into a queue for further processing. In particular for
+    sending the logs events to Splight GRPC API.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        access_id: str,
+        secret_key: str,
+        process_id: str,
+        sender_type: str,
+    ):
+        self._streamer = ComponentLogsStreamer(
+            host=host,
+            access_id=access_id,
+            secret_key=secret_key,
+            process_id=process_id,
+        )
+        self._sender_type = sender_type
+        self._streamer.start()
+
+    def save_record(self, record: LogRecord) -> None:
+        """Saves the logging record to be sent.
+
+        Parameters
+        ----------
+        record: LogRecord
+            The log record.
+        """
+        exc_info = None
+        try:
+            message = str(record.msg) % record.args
+        except Exception as exc:
+            log(exc)
+            message = str(record.msg)
+
+        if record.exc_info:
+            exc_info = "".join(
+                traceback.format_exception(*record.exc_info)
+            ).replace('"', "'")
+            message = str(record.msg)
+        else:
+            exc_info = ""
+
+        # tags attribute is not default in logger
+        tags = None
+        if hasattr(record, "tags"):
+            tags = getattr(record, "tags")
+        event = {
+            "sender_type": self._sender_type,
+            "name": record.name,
+            "message": message,
+            "loglevel": record.levelname,
+            "filename": record.filename,
+            "traceback": exc_info,
+            "tags": tags,
+        }
+        self._streamer.insert_message(event)
+
+    def flush(self) -> None:
+        """Flushes the log records."""
+        self._streamer.flush()
+
+    def stop(self) -> None:
+        """Stops the thread."""
+        self._streamer.stop()
+
+
+interceptor = ApplicationLogInterceptor(
+    host=settings.splight_platform_api_host,
+    access_id=settings.access_id,
+    secret_key=settings.secret_key,
+    process_id=settings.process_id,
+    sender_type=settings.process_type,
+)

splight_runner/api/settings.py

@@ -0,0 +1,63 @@
+import os
+from dataclasses import dataclass, field
+from functools import partial
+from typing import List
+
+SECRET_DIR = "/etc/config"
+
+
+class MissingEnvVariable(Exception):
+    ...
+
+
+def load_variable(var_name: str) -> str:
+    secret_file = os.path.join(SECRET_DIR, var_name)
+    if os.path.exists(secret_file):
+        with open(secret_file) as f:
+            variable = f.read().strip()
+    elif os.getenv(var_name):
+        variable = os.getenv(var_name)
+    else:
+        raise MissingEnvVariable(f"{var_name} is missing")
+    return variable
+
+
+def load_multiple_variable(var_names: List[str]) -> str:
+    value = None
+    for var_name in var_names:
+        try:
+            value = load_variable(var_name)
+            break
+        except MissingEnvVariable:
+            continue
+
+    if value is None:
+        raise MissingEnvVariable(f"On of {var_names} is missing")
+    return value
+
+
+@dataclass
+class SplightSettings:
+    """Class for holding Splight Runner settings."""
+
+    access_id: str = field(
+        default_factory=partial(load_variable, "SPLIGHT_ACCESS_ID")
+    )
+    secret_key: str = field(
+        default_factory=partial(load_variable, "SPLIGHT_SECRET_KEY")
+    )
+    splight_platform_api_host: str = field(
+        default_factory=partial(load_variable, "SPLIGHT_PLATFORM_API_HOST")
+    )
+    process_id: str = field(
+        default_factory=partial(
+            load_multiple_variable,
+            ["COMPONENT_ID", "COMPUTE_NODE_ID", "AGENT_ID"],
+        )
+    )
+    process_type: str = field(
+        default_factory=partial(load_variable, "PROCESS_TYPE")
+    )
+
+
+settings = SplightSettings()

splight_runner/bootstrap/sitecustomize.py

@@ -0,0 +1,39 @@
+# This file is used for configuring splight runner in runtime. In particular
+# loads some env vars and activate hooks for sending logs.
+# All hooks or configurations for the Splight Runner should be loaded in
+# this file.
+
+import ast
+import os
+import sys
+
+runner_active = ast.literal_eval(os.getenv("SPLIGHT_RUNNER_ACTIVE", "False"))
+
+boot_directory = os.path.dirname(__file__)
+root_directory = os.path.dirname(os.path.dirname(boot_directory))
+
+path = list(sys.path)
+
+if boot_directory in path:
+    del path[path.index(boot_directory)]
+
+try:
+    from importlib.machinery import PathFinder
+
+    module_spec = PathFinder.find_spec("sitecustomize", path=path)
+except ImportError as exc:
+    sys.stdout.write("Unable to import Splight Runner sitecustomie", exc)
+    sys.stdout.flush()
+else:
+    if module_spec is not None:
+        module_spec.loader.load_module("sitecustomize")
+
+if runner_active:
+    do_insert = root_directory not in sys.path
+    if do_insert:
+        sys.path.insert(0, root_directory)
+
+    import splight_runner.config
+
+    if do_insert:
+        del sys.path[sys.path.index(root_directory)]

splight_runner/commands/admin.py

@@ -0,0 +1,136 @@
+import inspect
+import sys
+from typing import Any, Callable, Dict, List, Optional
+
+
+class MissingCommandError(Exception):
+    """Exception"""
+
+    ...
+
+
+class MissingArgument(Exception):
+    """Exception raises when there is a missing argument for a command"""
+
+    ...
+
+
+class CommandAdmin:
+    """Class respobsible for taking registry of all the commands and callbaks"""
+
+    def __init__(self, name: str, version: str):
+        """Constructor
+
+        Parameters
+        ----------
+        name : str
+            Name of the application
+        version : str
+            The application version
+        """
+        self._name = name
+        self._version = version
+        self._commands: Dict[str, Callable] = {}
+        self._callback: Dict[str, Callable] = {}
+
+    def command(self, name: Optional[str] = None) -> Callable:
+        """Decorator to register a command
+
+        Parameters
+        ----------
+        name: Optional[str]
+            The name for the command
+
+        Returns
+        -------
+        Callable
+            The decorated function
+        """
+
+        def decorator(func: Callable) -> Callable:
+            func_name = name or func.__name__
+            self._commands.update({func_name: func})
+            return func
+
+        return decorator
+
+    def callback(self, name: Optional[str] = None) -> Callable:
+        """Decorator to register a callback
+
+        Parameters
+        ----------
+        name: Optional[str]
+            The name for the command
+
+        Returns
+        -------
+        Callable
+            The decorated function
+        """
+
+        def decorator(func: Callable) -> Callable:
+            func_name = name or func.__name__
+            self._callback.update({func_name: func})
+            return func
+
+        return decorator
+
+    def print_commands(self) -> None:
+        """Prints the commands and callbacks"""
+        title = f"{self._name} - {self._version}\n"
+        sys.stdout.write(title)
+        sys.stdout.write("{:=<{}}\n".format("", len(title)))
+        sys.stdout.write("Commands:\n")
+        for command in self._commands:
+            sys.stdout.write(f"{command: >20}\n")
+        sys.stdout.write("Callbacks:\n")
+        for callback in self._callback:
+            name = f"--{callback}"
+            sys.stdout.write(f"{name: >20}\n")
+
+    def __call__(self, *args: Any, **kwargs: Any):
+        """Dunder method to run a given command."""
+        if len(sys.argv) == 1:
+            raise MissingCommandError("No command provided")
+
+        command_name = sys.argv[1]
+        raw_args = sys.argv[2:]
+        if command_name.startswith("--"):
+            command = self._retrieve_callback(command_name.lstrip("--"))
+        else:
+            command = self._retrieve_command(command_name)
+
+        signature = inspect.signature(command)
+        command_args = self._parse_arguments(raw_args, signature=signature)
+        command(**command_args)
+
+    def _parse_arguments(
+        self, raw_args: List, signature: inspect.Signature
+    ) -> Dict:
+        arguments = {}
+        for idx, (name, parameter) in enumerate(signature.parameters.items()):
+            try:
+                raw_value = raw_args[idx]
+            except IndexError as exc:
+                raise MissingArgument(f"Missing argument {name}") from exc
+
+            argument = (
+                raw_value.split("=")[-1] if "=" in raw_value else raw_value
+            )
+            arg_value = parameter.annotation(argument)
+            arguments.update({name: arg_value})
+        return arguments
+
+    def _retrieve_command(self, command_name: str) -> Callable:
+        try:
+            command = self._commands[command_name]
+        except KeyError as exc:
+            sys.stderr.write(f"Command {command_name} not found: {exc}")
+        return command
+
+    def _retrieve_callback(self, command_name: str) -> Callable:
+        try:
+            command = self._callback[command_name]
+        except KeyError as exc:
+            sys.stderr.write(f"Callback {command_name} not found: {exc}")
+        return command

splight_runner/commands/execute_agent.py

@@ -0,0 +1,43 @@
+import os
+
+from splight_runner import __file__ as root_file
+from splight_runner.api.settings import settings
+from splight_runner.logging import log
+
+
+def execute_agent() -> None:
+    """Executes an splight agent after configuring splight runner hooks."""
+
+    root_dir = os.path.dirname(root_file)
+    boot_dir = os.path.join(root_dir, "bootstrap")
+
+    python_path = boot_dir
+    if "PYTHONPATH" in os.environ:
+        all_paths = os.getenv("PYTHONPATH").split(os.path.pathsep)
+        if boot_dir not in all_paths:
+            all_paths.insert(0, boot_dir)
+            python_path = os.path.pathsep.join(all_paths)
+
+    log("Configuring PYTHONPATH env var")
+    os.environ["PYTHONPATH"] = python_path
+    # The following variable is used for activate hooks
+    os.environ["SPLIGHT_RUNNER_ACTIVE"] = "True"
+
+    # Configure env variables for running Splight Agent
+    if "SPLIGHT_ACCESS_ID" not in os.environ:
+        os.environ["SPLIGHT_ACCESS_ID"] = settings.access_id
+    if "SPLIGHT_SECRET_KEY" not in os.environ:
+        os.environ["SPLIGHT_SECRET_KEY"] = settings.secret_key
+    if "SPLIGHT_PLATFORM_API_HOST" not in os.environ:
+        os.environ[
+            "SPLIGHT_PLATFORM_API_HOST"
+        ] = settings.splight_platform_api_host
+    if "COMPUTE_NODE_ID" not in os.environ:
+        os.environ["COMPUTE_NODE_ID"] = settings.process_id
+
+    log("Executing Splight Agent")
+
+    command = ["/bin/bash", "-c", "splight-agent"]
+    exec_path = "/bin/bash"
+
+    os.execl(exec_path, *command)

splight_runner/commands/execute_component.py

@@ -0,0 +1,84 @@
+import os
+import sys
+from pathlib import Path
+
+from splight_runner import __file__ as root_file
+from splight_runner.api.settings import settings
+
+
+class InvalidComponentLanguage(Exception):
+    ...
+
+
+class ComponentDoesNotExists(Exception):
+    ...
+
+
+def execute_component(component_file: Path, component_id: str) -> None:
+    """Executes a component after configuring splight runner hooks.
+
+    Parameters
+    ----------
+    component_file: Path
+        The path to the main file for the component.
+    component_id: str
+        The ID for the component to be executed.
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    ComponentDoesNotExists
+        If the component file does not exists.
+    InvalidComponentLanguage
+        If the component is written in a non-supported language.
+    """
+    # TODO: Add some logs messages for debugging
+    main_file = component_file.name
+    component_path = component_file.parent.resolve()
+    if not component_file.exists():
+        raise ComponentDoesNotExists("Component does not exists")
+    if main_file.endswith(".py"):
+        base_cmd = "python3"
+    else:
+        raise InvalidComponentLanguage(
+            "Component is not written in a supported language"
+        )
+
+    root_dir = os.path.dirname(root_file)
+    boot_dir = os.path.join(root_dir, "bootstrap")
+
+    python_path = boot_dir
+    if "PYTHONPATH" in os.environ:
+        all_paths = os.getenv("PYTHONPATH").split(os.path.pathsep)
+        if boot_dir not in all_paths:
+            all_paths.insert(0, boot_dir)
+            python_path = os.path.pathsep.join(all_paths)
+
+    os.environ["PYTHONPATH"] = python_path
+    # The following variable is used for activate hooks
+    os.environ["SPLIGHT_RUNNER_ACTIVE"] = "True"
+
+    # TODO: improve how env variables are configured
+    if "SPLIGHT_ACCESS_ID" not in os.environ:
+        os.environ["SPLIGHT_ACCESS_ID"] = settings.access_id
+    if "SPLIGHT_SECRET_KEY" not in os.environ:
+        os.environ["SPLIGHT_SECRET_KEY"] = settings.secret_key
+    if "SPLIGHT_PLATFORM_API_HOST" not in os.environ:
+        os.environ[
+            "SPLIGHT_PLATFORM_API_HOST"
+        ] = settings.splight_platform_api_host
+    if "COMPONENT_ID" not in os.environ:
+        os.environ["COMPONENT_ID"] = settings.process_id
+
+    os.chdir(component_path)
+    exec_path = sys.executable
+    command = [
+        base_cmd,
+        main_file,
+        "--component-id",
+        component_id,
+    ]
+    os.execl(exec_path, *command)

splight_runner/config.py

@@ -0,0 +1,72 @@
+import logging
+from types import ModuleType
+
+try:
+    import splight_lib
+
+    SPL_LIB_INSTALLED = True
+except Exception as exc:
+    print("Error importing splight_lib", exc)
+    SPL_LIB_INSTALLED = False
+
+from splight_runner.hooks import copy_module, on_import, reload_module
+from splight_runner.logging import log
+from splight_runner.wrapper.healthcheck import healthcheck_wrapper
+from splight_runner.wrapper.hooks import finish_execution
+from splight_runner.wrapper.logging import call_handlers_wrapper
+
+
+@on_import("logging")
+def on_logging_import(module: ModuleType) -> ModuleType:
+    """Hook for modifying default behavior for the logging module.
+
+    Parameters
+    ----------
+    module: ModuleType
+        The built-in logging module
+
+    Returns
+    -------
+    ModuleType: The logging module updated
+    """
+    original = getattr(module.Logger, "callHandlers", None)
+    if original:
+        wrapped = call_handlers_wrapper(original)
+        new_module = copy_module(logging)
+        setattr(new_module.Logger, "callHandlers", wrapped)
+        return new_module
+    return module
+
+
+@on_import("splight_lib")
+def on_splight_lib_import(module: ModuleType) -> ModuleType:
+    if hasattr(module.execution, "ExecutionClient"):
+        original = getattr(
+            module.execution.ExecutionClient, "healthcheck", None
+        )
+        engine_name = "ExecutionClient"
+    elif hasattr(module.execution, "ExecutionEngine"):
+        original = getattr(
+            module.execution.ExecutionEngine, "healthcheck", None
+        )
+        engine_name = "ExecutionEngine"
+    else:
+        log("Unable to find Execution Engine in splight_lib")
+        return module
+    if original and SPL_LIB_INSTALLED:
+        wrapped = healthcheck_wrapper(original)
+        new_module = copy_module(splight_lib)
+        engine = getattr(module.execution, engine_name)
+        setattr(engine, "healthcheck", wrapped)
+        return new_module
+    return module
+
+
+# Reload logging module to update everywhere
+reload_module(logging)
+
+
+if SPL_LIB_INSTALLED:
+    reload_module(splight_lib)
+
+finish_execution()