pyxecm 1.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

pyxecm/customizer/api/common/router.py

@@ -0,0 +1,72 @@
+"""API Implemenation for the Customizer to start and control the payload processing."""
+
+import logging
+import os
+import signal
+from http import HTTPStatus
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
+from pyxecm.customizer.api.common.models import CustomizerStatus
+
+router = APIRouter(tags=["default"])
+
+logger = logging.getLogger("pyxecm.customizer.api.common")
+
+
+@router.get("/", include_in_schema=False)
+async def redirect_to_api() -> RedirectResponse:
+    """Redirect from / to /api.
+
+    Returns:
+        None
+
+    """
+    return RedirectResponse(url="/api")
+
+
+@router.get(path="/status", name="Get Status")
+async def get_status() -> CustomizerStatus:
+    """Get the status of the Customizer."""
+
+    df = PAYLOAD_LIST.get_payload_items()
+
+    if df is None:
+        raise HTTPException(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            detail="Payload list is empty.",
+        )
+
+    all_status = df["status"].value_counts().to_dict()
+
+    return CustomizerStatus(
+        version=2,
+        customizer_duration=(all_status.get("running", None)),
+        customizer_end_time=None,
+        customizer_start_time=None,
+        status_details=all_status,
+        status="Running" if "running" in all_status else "Stopped",
+        debug=df["log_debug"].sum(),
+        info=df["log_info"].sum(),
+        warning=df["log_warning"].sum(),
+        error=df["log_error"].sum(),
+        critical=df["log_critical"].sum(),
+    )
+
+
+@router.get("/api/shutdown", include_in_schema=False)
+def shutdown(user: Annotated[User, Depends(get_authorized_user)]) -> JSONResponse:
+    """Endpoint to end the application."""
+
+    logger.warning(
+        "Shutting down the API - Requested via api by user -> %s",
+        user.id,
+    )
+    os.kill(os.getpid(), signal.SIGTERM)
+
+    return JSONResponse({"status": "shutdown"}, status_code=HTTPStatus.ACCEPTED)

pyxecm/customizer/api/settings.py

@@ -0,0 +1,128 @@
+"""Settings for Customizer execution."""
+
+import os
+import tempfile
+import uuid
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import (
+    BaseSettings,
+    SettingsConfigDict,
+)
+
+
+## Customzer Settings
+class CustomizerAPISettings(BaseSettings):
+    """Settings for the Customizer API."""
+
+    api_token: str | None = Field(
+        default=None,
+        description="Optional token that can be specified that has access to the Customizer API, bypassing the OTDS authentication.",
+    )
+    bind_address: str = Field(default="0.0.0.0", description="Interface to bind the Customizer API.")  # noqa: S104
+    bind_port: int = Field(default=8000, description="Port to bind the Customizer API to")
+    workers: int = Field(default=1, description="Number of workers to use for the API BackgroundTasks")
+
+    import_payload: bool = Field(default=False)
+    payload: str = Field(
+        default="/payload/payload.yml.gz.b64",
+        description="Path to a single Payload file to be loaded.",
+    )
+    payload_dir: str = Field(
+        default="/payload-external/",
+        description="Path to a directory of Payload files. All files in this directory will be loaded in alphabetical order and dependencies will be added automatically on the previous object. So all payload in this folder will be processed sequentially in alphabetical oder.",
+    )
+    payload_dir_optional: str = Field(
+        default="/payload-optional/",
+        description="Path of Payload files to be loaded. No additional logic for dependencies will be applied, they need to be managed within the payloadSetitings section of each payload. See -> payloadOptions in the Payload Syntax documentation.",
+    )
+
+    temp_dir: str = Field(
+        default=os.path.join(tempfile.gettempdir(), "customizer"),
+        description="location of the temp folder. Used for temporary files during the payload execution",
+    )
+
+    loglevel: Literal["INFO", "DEBUG", "WARNING", "ERROR"] = "INFO"
+    logfolder: str = Field(
+        default=os.path.join(tempfile.gettempdir(), "customizer"),
+        description="Logfolder for Customizer logfiles",
+    )
+    logfile: str = Field(
+        default="customizer.log",
+        description="Logfile for Customizer API. This logfile also contains the execution of every payload.",
+    )
+
+    namespace: str = Field(
+        default="default",
+        description="Namespace to use for otxecm resource lookups",
+    )
+    maintenance_mode: bool = Field(
+        default=True,
+        description="Automatically enable and disable the maintenance mode during payload deployments.",
+    )
+
+    trusted_origins: list[str] = Field(
+        default=[
+            "http://localhost",
+            "http://localhost:5173",
+            "http://localhost:8080",
+            "https://manager.develop.terrarium.cloud",
+            "https://manager.terrarium.cloud",
+        ],
+    )
+
+    otds_protocol: str = Field(default="http", alias="OTDS_PROTOCOL")
+    otds_host: str = Field(default="otds", alias="OTDS_HOSTNAME")
+    otds_port: int = Field(default=80, alias="OTDS_SERVICE_PORT_OTDS")
+    otds_url: str | None = Field(default=None, alias="OTDS_URL")
+
+    metrics: bool = Field(
+        default=True,
+        description="Enable or disable the /metrics endpoint for Prometheus",
+    )
+
+    victorialogs_host: str = Field(
+        default="",
+        description="Hostname of the VictoriaLogs Server",
+    )
+    victorialogs_port: int = Field(
+        default=9428,
+        description="Port of the VictoriaLogs Server",
+    )
+
+    csai: bool = Field(
+        default=True,
+        description="Enable the CSAI integration",
+    )
+
+    csai_prefix: str = Field(
+        default="csai",
+        description="Prefix for the CSAI",
+    )
+
+    upload_folder: str = Field(default=os.path.join(tempfile.gettempdir(), "upload"), description="Folder for uploads")
+
+    upload_key: str = Field(default=str(uuid.uuid4()), description="Upload key for the Logs")
+
+    upload_url: str = Field(
+        default="http://customizer:8000/api/v1/otcs/logs/upload", description="Upload URL for the Logs"
+    )
+
+    ws_terminal: bool = Field(
+        default=False,
+        description="Enable the Websocket Terminal endpoint",
+    )
+
+    model_config = SettingsConfigDict(env_prefix="CUSTOMIZER_")
+
+    def __init__(self, **data: any) -> None:
+        """Class initializer."""
+
+        super().__init__(**data)
+        if self.otds_url is None:
+            self.otds_url = f"{self.otds_protocol}://{self.otds_host}:{self.otds_port}"
+
+
+# Create Instance of settings
+api_settings = CustomizerAPISettings()

pyxecm/customizer/api/terminal/__init__.py

@@ -0,0 +1 @@
+"""WebSocket Terminal Server defintion."""

pyxecm/customizer/api/terminal/router.py

@@ -0,0 +1,87 @@
+"""Define a websocket to connect to a shell session in a pod."""
+
+import asyncio
+import contextlib
+import os
+import pty
+import signal
+import subprocess
+
+from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect, status
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user, get_current_user
+
+router = APIRouter(tags=["terminal"])
+
+
+@router.websocket("/ws/terminal")
+async def ws_terminal(websocket: WebSocket, pod: str = Query(...), command: str = Query(...)) -> None:
+    """Websocket to connect to a shell session in a pod.
+
+    Args:
+        websocket (WebSocket): WebSocket to connect to the shell session.
+        pod (str): pod name to connect to.
+        command (str): command to be executed.
+
+    """
+    await websocket.accept()
+
+    try:
+        # Wait for the first message to be the token
+        token = await websocket.receive_text()
+
+        user = await get_current_user(token)
+        authrorized = await get_authorized_user(user)
+
+        if not authrorized:
+            await websocket.send_text("Invalid User: " + str(user))
+            await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
+            return
+
+    except HTTPException:
+        await websocket.send_text("Invalid Token")
+        await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
+
+    except WebSocketDisconnect:
+        return
+
+    process = ["bash"] if pod == "customizer" else ["kubectl", "exec", "-it", pod, "--", command]
+
+    pid, fd = pty.fork()
+    if pid == 0:
+        subprocess.run(process, check=False)  # noqa: ASYNC221
+
+    async def read_from_pty() -> None:
+        loop = asyncio.get_event_loop()
+        try:
+            while True:
+                data = await loop.run_in_executor(None, os.read, fd, 1024)
+                await websocket.send_text(data.decode(errors="ignore"))
+        except Exception:  # noqa: S110
+            pass  # PTY closed or WebSocket failed
+
+    async def write_to_pty() -> None:
+        try:
+            while True:
+                data = await websocket.receive_text()
+                os.write(fd, data.encode())
+        except Exception:  # noqa: S110
+            pass
+
+    # Launch read/write tasks
+    read_task = asyncio.create_task(read_from_pty())
+    write_task = asyncio.create_task(write_to_pty())
+
+    done, pending = await asyncio.wait([read_task, write_task], return_when=asyncio.FIRST_COMPLETED)
+
+    # Cancel other task
+    for task in pending:
+        task.cancel()
+
+    try:  # noqa: SIM105
+        os.kill(pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass  # Already exited
+
+    with contextlib.suppress(Exception):
+        os.close(fd)

pyxecm/customizer/api/v1_csai/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_csai/router.py

@@ -0,0 +1,87 @@
+"""Define router for v1_maintenance."""
+
+import logging
+from http import HTTPStatus
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+from fastapi.responses import JSONResponse
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.common.functions import get_k8s_object, get_settings
+from pyxecm.customizer.api.settings import CustomizerAPISettings
+from pyxecm.customizer.k8s import K8s
+
+router = APIRouter(prefix="/api/v1/csai", tags=["csai"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_csai")
+
+
+@router.get("")
+def get_csai_config_data(
+    user: Annotated[User, Depends(get_authorized_user)],
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+) -> JSONResponse:
+    """Get the csai config data."""
+
+    logger.info("READ csai config data by user -> %s", user.id)
+
+    config_data = {}
+
+    try:
+        csai_config_maps = [
+            cm for cm in k8s_object.list_config_maps().items if cm.metadata.name.startswith(settings.csai_prefix)
+        ]
+
+        for config_map in csai_config_maps:
+            config_data[config_map.metadata.name] = config_map.data
+
+    except Exception as e:
+        logger.error("Could not read config data from k8s -> %s", e)
+        return JSONResponse({"status": "error", "message": str(e)})
+
+    return JSONResponse(config_data)
+
+
+@router.post("")
+def set_csai_config_data(
+    user: Annotated[User, Depends(get_authorized_user)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    config: Annotated[dict, Body()],
+) -> JSONResponse:
+    """Get the csai config data."""
+
+    logger.info("READ csai config data by user -> %s", user.id)
+
+    for config_map in config:
+        if not config_map.startswith(settings.csai_prefix):
+            return JSONResponse(
+                {
+                    "status": "error",
+                    "message": f"Config map name {config_map} does not start with {settings.csai_prefix}",
+                },
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+
+    try:
+        for key, value in config.items():
+            logger.info("User: %s -> Replacing config map %s with %s", user.id, key, value)
+            k8s_object.replace_config_map(
+                config_map_name=key,
+                config_map_data=value,
+            )
+
+    except Exception as e:
+        logger.error("Could not replace config map %s with %s -> %s", key, value, e)
+        return JSONResponse({"status": "error", "message": str(e)})
+
+    for deployment in ["chat-svc", "embed-svc", "embed-wrkr"]:
+        deployment = f"{settings.csai_prefix}-{deployment}"
+
+        logger.info("User: %s ->Restarting deployment -> %s", user.id, deployment)
+        k8s_object.restart_deployment(deployment)
+
+    return get_csai_config_data(user=user, k8s_object=k8s_object, settings=settings)

pyxecm/customizer/api/v1_maintenance/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_maintenance/functions.py

@@ -0,0 +1,100 @@
+"""Define functions for v1_maintenance."""
+
+import logging
+import os
+from http import HTTPStatus
+
+import yaml
+from fastapi import HTTPException
+from pydantic import HttpUrl, ValidationError
+
+from pyxecm.customizer.api.v1_maintenance.models import MaintenanceModel
+from pyxecm.customizer.k8s import K8s
+from pyxecm.maintenance_page.settings import settings as maint_settings
+
+logger = logging.getLogger("v1_maintenance.functions")
+
+
+def get_cshost(k8s_object: K8s) -> str:
+    """Get the cs_hostname from the environment Variable OTCS_PUBLIC_HOST otherwise read it from the otcs-frontend-configmap."""
+
+    if "OTCS_PUBLIC_URL" in os.environ:
+        return os.getenv("OTCS_PUBLIC_URL", "otcs")
+
+    else:
+        cm = k8s_object.get_config_map("otcs-frontend-configmap")
+
+        if cm is None:
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail=f"Could not read otcs-frontend-configmap from namespace: {k8s_object.get_namespace()}",
+            )
+
+        config_file = cm.data.get("config.yaml")
+        config = yaml.safe_load(config_file)
+
+        try:
+            cs_url = HttpUrl(config.get("csurl"))
+        except ValidationError as ve:
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail="Could not read otcs_host from environment variable OTCS_PULIBC_URL or configmap otcs-frontend-configmap/config.yaml/cs_url",
+            ) from ve
+        return cs_url.host
+
+
+def get_maintenance_mode_status(k8s_object: K8s) -> dict:
+    """Get status of maintenance mode.
+
+    Returns:
+        dict:
+            Details of maintenance mode.
+
+    """
+    ingress = k8s_object.get_ingress("otxecm-ingress")
+
+    if ingress is None:
+        raise HTTPException(
+            status_code=500,
+            detail="No ingress object found to read Maintenance Mode status",
+        )
+
+    enabled = False
+    for rule in ingress.spec.rules:
+        if rule.host == get_cshost(k8s_object):
+            enabled = rule.http.paths[0].backend.service.name != "otcs-frontend"
+
+    return MaintenanceModel(
+        enabled=enabled, title=maint_settings.title, text=maint_settings.text, footer=maint_settings.footer
+    )
+
+    # return {
+    #     "enabled": enabled,
+    #     "title": maint_settings.title,
+    #     "text": maint_settings.text,
+    #     "footer": maint_settings.footer,
+    # }
+
+
+def set_maintenance_mode_via_ingress(enabled: bool, k8s_object: K8s) -> None:
+    """Set maintenance mode."""
+
+    logger.warning(
+        "Setting Maintenance Mode to -> %s",
+        (enabled),
+    )
+
+    if enabled:
+        k8s_object.update_ingress_backend_services(
+            "otxecm-ingress",
+            get_cshost(k8s_object=k8s_object),
+            "customizer",
+            5555,
+        )
+    else:
+        k8s_object.update_ingress_backend_services(
+            "otxecm-ingress",
+            get_cshost(k8s_object=k8s_object),
+            "otcs-frontend",
+            80,
+        )

pyxecm/customizer/api/v1_maintenance/models.py

@@ -0,0 +1,12 @@
+"""Define Models for the Maintenance Page Config."""
+
+from pydantic import BaseModel
+
+
+class MaintenanceModel(BaseModel):
+    """Status object of the Maintenance Page."""
+
+    enabled: bool
+    title: str | None = ""
+    text: str | None = ""
+    footer: str | None = ""

pyxecm/customizer/api/v1_maintenance/router.py

@@ -0,0 +1,76 @@
+"""Define router for v1_maintenance."""
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form
+
+from pyxecm.customizer.api.auth import models
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.common.functions import get_k8s_object
+from pyxecm.customizer.api.v1_maintenance.functions import get_maintenance_mode_status, set_maintenance_mode_via_ingress
+from pyxecm.customizer.api.v1_maintenance.models import MaintenanceModel
+from pyxecm.customizer.k8s import K8s
+from pyxecm.maintenance_page.settings import settings as maint_settings
+
+router = APIRouter(prefix="/api/v1/maintenance", tags=["maintenance"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_maintenance")
+
+
+@router.get(path="")
+async def status(
+    user: Annotated[models.User, Depends(get_authorized_user)],  # noqa: ARG001
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+) -> MaintenanceModel:
+    """Return status of maintenance mode.
+
+    Args:
+        user (models.User):
+            Added to enforce authentication requirement
+        k8s_object (K8s):
+            K8s object instance of pyxecm K8s class
+
+
+    Returns:
+        dict:
+            Details of maintenance mode.
+
+    """
+
+    return get_maintenance_mode_status(k8s_object)
+
+
+@router.post(path="")
+async def set_maintenance_mode_options(
+    user: Annotated[models.User, Depends(get_authorized_user)],  # noqa: ARG001
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    config: Annotated[MaintenanceModel, Form()],
+) -> MaintenanceModel:
+    """Configure the Maintenance Mode and set options.
+
+    Args:
+        user (models.User):
+            Added to enforce authentication requirement
+        k8s_object (K8s):
+            K8s object instance of pyxecm K8s class
+        config (MaintenanceModel):
+            instance of the Maintenance Model
+
+    Returns:
+        dict: _description_
+
+    """
+    # Enable / Disable the acutual Maintenance Mode
+    set_maintenance_mode_via_ingress(config.enabled, k8s_object)
+
+    if config.title:
+        maint_settings.title = config.title
+
+    if config.text:
+        maint_settings.text = config.text
+
+    if config.footer:
+        maint_settings.footer = config.footer
+
+    return get_maintenance_mode_status(k8s_object)

pyxecm/customizer/api/v1_otcs/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_otcs/functions.py

@@ -0,0 +1,61 @@
+"""Define functions for v1_otcs."""
+
+import logging
+from datetime import datetime, timezone
+from threading import Lock
+
+from fastapi import APIRouter
+
+from pyxecm.customizer.api.settings import CustomizerAPISettings
+from pyxecm.customizer.k8s import K8s
+
+router = APIRouter(prefix="/api/v1/otcs", tags=["otcs"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_otcs")
+
+
+def collect_otcs_logs(host: str, k8s_object: K8s, logs_lock: Lock, settings: CustomizerAPISettings) -> None:
+    """Collect the logs for the given OTCS instance."""
+
+    with logs_lock:
+        timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d_%H-%M")
+        tgz_file = f"/tmp/{timestamp}_{host}.tar.gz"  # noqa: S108
+
+        if host.startswith("otcs-frontend"):
+            container = "otcs-frontend-container"
+        elif host.startswith("otcs-backend-search"):
+            container = "otcs-backend-search-container"
+        elif host.startswith("otcs-admin"):
+            container = "otcs-admin-container"
+        else:
+            container = None
+
+        logger.info("Collecting logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=["tar", "-czvf", tgz_file, "/opt/opentext/cs/logs", "/opt/opentext/cs_persist/contentserver.log"],
+            container=container,
+            timeout=1800,
+        )
+
+        logger.info("Uploading logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=[
+                "curl",
+                "-X",
+                "POST",
+                "-F",
+                f"file=@{tgz_file}",
+                f"{settings.upload_url}?key={settings.upload_key}",
+            ],
+            container=container,
+            timeout=1800,
+        )
+
+        logger.info("Cleanup logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=["rm", tgz_file],
+            container=container,
+        )