pyxecm 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

pyxecm/customizer/api/auth/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/{auth.py → auth/functions.py}

@@ -5,10 +5,9 @@ from typing import Annotated
 
 import requests
 from fastapi import APIRouter, Depends, HTTPException, status
-from fastapi.responses import JSONResponse
-from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from fastapi.security import OAuth2PasswordBearer
 
-from pyxecm.customizer.api.models import User
+from pyxecm.customizer.api.auth.models import User
 from pyxecm.customizer.api.settings import api_settings
 
 router = APIRouter()
@@ -91,64 +90,3 @@ async def get_authorized_user(current_user: Annotated[User, Depends(get_current_
             detail=f"User {current_user.id} is not authorized",
         )
     return current_user
-
-
-@router.post("/token", tags=["auth"])
-async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> JSONResponse:
-    """Login using OTDS and return a token."""
-
-    url = api_settings.otds_url + "/otdsws/rest/authentication/credentials"
-
-    payload = json.dumps(
-        {"userName": form_data.username, "password": form_data.password},
-    )
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-    }
-
-    try:
-        response = requests.request(
-            "POST",
-            url,
-            headers=headers,
-            data=payload,
-            timeout=10,
-        )
-    except requests.exceptions.ConnectionError as exc:
-        raise HTTPException(
-            status_code=500,
-            detail=f"{exc.request.url} cannot be reached",
-        ) from exc
-
-    if response.ok:
-        response = json.loads(response.text)
-    else:
-        raise HTTPException(status_code=400, detail="Incorrect username or password")
-
-    return JSONResponse(
-        {
-            "access_token": response["ticket"],
-            "token_type": "bearer",
-            "userId": response["userId"],
-        },
-    )
-
-
-@router.get("/users/me", tags=["auth"])
-async def read_users_me(current_user: Annotated[User, Depends(get_current_user)]) -> JSONResponse:
-    """Get the current user.
-
-    current_user:
-        type: User
-        description: The current user.
-
-    """
-
-    if "otadmins@otds.admin" in current_user.groups:
-        return JSONResponse(current_user.model_dump())
-    else:
-        raise HTTPException(
-            status_code=403,
-            detail=f"User {current_user.id} is not authorized",
-        )

pyxecm/customizer/api/auth/router.py

@@ -0,0 +1,78 @@
+"""Utility library to handle the authentication with OTDS."""
+
+import json
+from typing import Annotated
+
+import requests
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+
+from pyxecm.customizer.api.auth.functions import get_current_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.settings import api_settings
+
+router = APIRouter()
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+
+
+@router.post("/token", tags=["auth"])
+async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> JSONResponse:
+    """Login using OTDS and return a token."""
+
+    url = api_settings.otds_url + "/otdsws/rest/authentication/credentials"
+
+    payload = json.dumps(
+        {"userName": form_data.username, "password": form_data.password},
+    )
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    try:
+        response = requests.request(
+            "POST",
+            url,
+            headers=headers,
+            data=payload,
+            timeout=10,
+        )
+    except requests.exceptions.ConnectionError as exc:
+        raise HTTPException(
+            status_code=500,
+            detail=f"{exc.request.url} cannot be reached",
+        ) from exc
+
+    if response.ok:
+        response = json.loads(response.text)
+    else:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+
+    return JSONResponse(
+        {
+            "access_token": response["ticket"],
+            "token_type": "bearer",
+            "userId": response["userId"],
+        },
+    )
+
+
+@router.get("/users/me", tags=["auth"])
+async def read_users_me(current_user: Annotated[User, Depends(get_current_user)]) -> JSONResponse:
+    """Get the current user.
+
+    current_user:
+        type: User
+        description: The current user.
+
+    """
+
+    if "otadmins@otds.admin" in current_user.groups:
+        return JSONResponse(current_user.model_dump())
+    else:
+        raise HTTPException(
+            status_code=403,
+            detail=f"User {current_user.id} is not authorized",
+        )

pyxecm/customizer/api/common/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/common/functions.py

@@ -0,0 +1,47 @@
+"""Define common functions."""
+
+import logging
+
+from pyxecm.customizer.api.common.payload_list import PayloadList
+from pyxecm.customizer.api.settings import CustomizerAPISettings, api_settings
+from pyxecm.customizer.k8s import K8s
+
+logger = logging.getLogger("pyxecm.customizer.api")
+
+# Create a LOCK dict for singleton logs collection
+LOGS_LOCK = {}
+# Initialize the globel Payloadlist object
+PAYLOAD_LIST = PayloadList(logger=logger)
+
+
+def get_k8s_object() -> K8s:
+    """Get an instance of a K8s object.
+
+    Returns:
+        K8s: Return a K8s object
+
+    """
+
+    return K8s(logger=logger, namespace=api_settings.namespace)
+
+
+def get_settings() -> CustomizerAPISettings:
+    """Get the API Settings object.
+
+    Returns:
+        CustomizerPISettings: Returns the API Settings
+
+    """
+
+    return api_settings
+
+
+def get_otcs_logs_lock() -> dict:
+    """Get the Logs LOCK dict.
+
+    Returns:
+        The dict with all LOCKS for the logs
+
+    """
+
+    return LOGS_LOCK

pyxecm/customizer/api/{metrics.py → common/metrics.py}

@@ -5,7 +5,7 @@ from collections.abc import Callable
 from prometheus_client import Gauge
 from prometheus_fastapi_instrumentator.metrics import Info
 
-from pyxecm.customizer.api.payload_list import PayloadList
+from pyxecm.customizer.api.common.payload_list import PayloadList
 
 
 ## By Payload

pyxecm/customizer/api/common/models.py

@@ -0,0 +1,21 @@
+"""Define common base Models."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class CustomizerStatus(BaseModel):
+    """Define Model for Customizer Status."""
+
+    version: int = 2
+    customizer_duration: Any | None
+    customizer_end_time: Any | None
+    customizer_start_time: Any | None
+    status_details: dict
+    status: str = "Stopped"
+    debug: int = 0
+    info: int = 0
+    warning: int = 0
+    error: int = 0
+    critical: int = 0

pyxecm/customizer/api/{payload_list.py → common/payload_list.py}

@@ -82,6 +82,7 @@ class PayloadList:
                 "log_warning",
                 "log_error",
                 "log_critical",
+                "customizer_settings",
             ],
         )
 
@@ -222,6 +223,7 @@ class PayloadList:
         enabled: bool = True,
         git_url: str | None = None,
         loglevel: str = "INFO",
+        customizer_settings: dict | None = None,
     ) -> dict:
         """Add a new item to the PayloadList.
 
@@ -243,6 +245,8 @@ class PayloadList:
                 Link to the payload in the GIT repository.
             loglevel (str):
                 The log level for processing the payload. Either "INFO" or "DEBUG".
+            customizer_settings (dict):
+                Customizer settings for the payload. Defaults to None.
 
         """
 
@@ -260,6 +264,7 @@ class PayloadList:
             "log_warning": 0,
             "log_error": 0,
             "log_critical": 0,
+            "customizer_settings": customizer_settings if customizer_settings else {},
         }
         self.payload_items = pd.concat(
             [self.payload_items, pd.DataFrame([new_item])],
@@ -694,7 +699,7 @@ class PayloadList:
                 success = False
 
             if payload:
-                customizer_settings = payload.get("customizerSettings", {})
+                customizer_settings = payload_item["customizer_settings"]
 
                 # Overwrite the customizer settings with the payload specific ones:
                 customizer_settings.update(

pyxecm/customizer/api/common/router.py

@@ -0,0 +1,72 @@
+"""API Implemenation for the Customizer to start and control the payload processing."""
+
+import logging
+import os
+import signal
+from http import HTTPStatus
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
+from pyxecm.customizer.api.common.models import CustomizerStatus
+
+router = APIRouter(tags=["default"])
+
+logger = logging.getLogger("pyxecm.customizer.api.common")
+
+
+@router.get("/", include_in_schema=False)
+async def redirect_to_api() -> RedirectResponse:
+    """Redirect from / to /api.
+
+    Returns:
+        None
+
+    """
+    return RedirectResponse(url="/api")
+
+
+@router.get(path="/status", name="Get Status")
+async def get_status() -> CustomizerStatus:
+    """Get the status of the Customizer."""
+
+    df = PAYLOAD_LIST.get_payload_items()
+
+    if df is None:
+        raise HTTPException(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            detail="Payload list is empty.",
+        )
+
+    all_status = df["status"].value_counts().to_dict()
+
+    return CustomizerStatus(
+        version=2,
+        customizer_duration=(all_status.get("running", None)),
+        customizer_end_time=None,
+        customizer_start_time=None,
+        status_details=all_status,
+        status="Running" if "running" in all_status else "Stopped",
+        debug=df["log_debug"].sum(),
+        info=df["log_info"].sum(),
+        warning=df["log_warning"].sum(),
+        error=df["log_error"].sum(),
+        critical=df["log_critical"].sum(),
+    )
+
+
+@router.get("/api/shutdown", include_in_schema=False)
+def shutdown(user: Annotated[User, Depends(get_authorized_user)]) -> JSONResponse:
+    """Endpoint to end the application."""
+
+    logger.warning(
+        "Shutting down the API - Requested via api by user -> %s",
+        user.id,
+    )
+    os.kill(os.getpid(), signal.SIGTERM)
+
+    return JSONResponse({"status": "shutdown"}, status_code=HTTPStatus.ACCEPTED)

pyxecm/customizer/api/settings.py

@@ -2,6 +2,7 @@
 
 import os
 import tempfile
+import uuid
 from typing import Literal
 
 from pydantic import Field
@@ -21,6 +22,7 @@ class CustomizerAPISettings(BaseSettings):
     )
     bind_address: str = Field(default="0.0.0.0", description="Interface to bind the Customizer API.")  # noqa: S104
     bind_port: int = Field(default=8000, description="Port to bind the Customizer API to")
+    workers: int = Field(default=1, description="Number of workers to use for the API BackgroundTasks")
 
     import_payload: bool = Field(default=False)
     payload: str = Field(
@@ -89,6 +91,29 @@ class CustomizerAPISettings(BaseSettings):
         description="Port of the VictoriaLogs Server",
     )
 
+    csai: bool = Field(
+        default=True,
+        description="Enable the CSAI integration",
+    )
+
+    csai_prefix: str = Field(
+        default="csai",
+        description="Prefix for the CSAI",
+    )
+
+    upload_folder: str = Field(default=os.path.join(tempfile.gettempdir(), "upload"), description="Folder for uploads")
+
+    upload_key: str = Field(default=str(uuid.uuid4()), description="Upload key for the Logs")
+
+    upload_url: str = Field(
+        default="http://customizer:8000/api/v1/otcs/logs/upload", description="Upload URL for the Logs"
+    )
+
+    ws_terminal: bool = Field(
+        default=False,
+        description="Enable the Websocket Terminal endpoint",
+    )
+
     model_config = SettingsConfigDict(env_prefix="CUSTOMIZER_")
 
     def __init__(self, **data: any) -> None:

pyxecm/customizer/api/terminal/__init__.py

@@ -0,0 +1 @@
+"""WebSocket Terminal Server defintion."""

pyxecm/customizer/api/terminal/router.py

@@ -0,0 +1,87 @@
+"""Define a websocket to connect to a shell session in a pod."""
+
+import asyncio
+import contextlib
+import os
+import pty
+import signal
+import subprocess
+
+from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect, status
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user, get_current_user
+
+router = APIRouter(tags=["terminal"])
+
+
+@router.websocket("/ws/terminal")
+async def ws_terminal(websocket: WebSocket, pod: str = Query(...), command: str = Query(...)) -> None:
+    """Websocket to connect to a shell session in a pod.
+
+    Args:
+        websocket (WebSocket): WebSocket to connect to the shell session.
+        pod (str): pod name to connect to.
+        command (str): command to be executed.
+
+    """
+    await websocket.accept()
+
+    try:
+        # Wait for the first message to be the token
+        token = await websocket.receive_text()
+
+        user = await get_current_user(token)
+        authrorized = await get_authorized_user(user)
+
+        if not authrorized:
+            await websocket.send_text("Invalid User: " + str(user))
+            await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
+            return
+
+    except HTTPException:
+        await websocket.send_text("Invalid Token")
+        await websocket.close(code=status.WS_1008_POLICY_VIOLATION)
+
+    except WebSocketDisconnect:
+        return
+
+    process = ["bash"] if pod == "customizer" else ["kubectl", "exec", "-it", pod, "--", command]
+
+    pid, fd = pty.fork()
+    if pid == 0:
+        subprocess.run(process, check=False)  # noqa: ASYNC221
+
+    async def read_from_pty() -> None:
+        loop = asyncio.get_event_loop()
+        try:
+            while True:
+                data = await loop.run_in_executor(None, os.read, fd, 1024)
+                await websocket.send_text(data.decode(errors="ignore"))
+        except Exception:  # noqa: S110
+            pass  # PTY closed or WebSocket failed
+
+    async def write_to_pty() -> None:
+        try:
+            while True:
+                data = await websocket.receive_text()
+                os.write(fd, data.encode())
+        except Exception:  # noqa: S110
+            pass
+
+    # Launch read/write tasks
+    read_task = asyncio.create_task(read_from_pty())
+    write_task = asyncio.create_task(write_to_pty())
+
+    done, pending = await asyncio.wait([read_task, write_task], return_when=asyncio.FIRST_COMPLETED)
+
+    # Cancel other task
+    for task in pending:
+        task.cancel()
+
+    try:  # noqa: SIM105
+        os.kill(pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass  # Already exited
+
+    with contextlib.suppress(Exception):
+        os.close(fd)

pyxecm/customizer/api/v1_csai/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_csai/router.py

@@ -0,0 +1,87 @@
+"""Define router for v1_maintenance."""
+
+import logging
+from http import HTTPStatus
+from typing import Annotated
+
+from fastapi import APIRouter, Body, Depends
+from fastapi.responses import JSONResponse
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.common.functions import get_k8s_object, get_settings
+from pyxecm.customizer.api.settings import CustomizerAPISettings
+from pyxecm.customizer.k8s import K8s
+
+router = APIRouter(prefix="/api/v1/csai", tags=["csai"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_csai")
+
+
+@router.get("")
+def get_csai_config_data(
+    user: Annotated[User, Depends(get_authorized_user)],
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+) -> JSONResponse:
+    """Get the csai config data."""
+
+    logger.info("READ csai config data by user -> %s", user.id)
+
+    config_data = {}
+
+    try:
+        csai_config_maps = [
+            cm for cm in k8s_object.list_config_maps().items if cm.metadata.name.startswith(settings.csai_prefix)
+        ]
+
+        for config_map in csai_config_maps:
+            config_data[config_map.metadata.name] = config_map.data
+
+    except Exception as e:
+        logger.error("Could not read config data from k8s -> %s", e)
+        return JSONResponse({"status": "error", "message": str(e)})
+
+    return JSONResponse(config_data)
+
+
+@router.post("")
+def set_csai_config_data(
+    user: Annotated[User, Depends(get_authorized_user)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    config: Annotated[dict, Body()],
+) -> JSONResponse:
+    """Get the csai config data."""
+
+    logger.info("READ csai config data by user -> %s", user.id)
+
+    for config_map in config:
+        if not config_map.startswith(settings.csai_prefix):
+            return JSONResponse(
+                {
+                    "status": "error",
+                    "message": f"Config map name {config_map} does not start with {settings.csai_prefix}",
+                },
+                status_code=HTTPStatus.BAD_REQUEST,
+            )
+
+    try:
+        for key, value in config.items():
+            logger.info("User: %s -> Replacing config map %s with %s", user.id, key, value)
+            k8s_object.replace_config_map(
+                config_map_name=key,
+                config_map_data=value,
+            )
+
+    except Exception as e:
+        logger.error("Could not replace config map %s with %s -> %s", key, value, e)
+        return JSONResponse({"status": "error", "message": str(e)})
+
+    for deployment in ["chat-svc", "embed-svc", "embed-wrkr"]:
+        deployment = f"{settings.csai_prefix}-{deployment}"
+
+        logger.info("User: %s ->Restarting deployment -> %s", user.id, deployment)
+        k8s_object.restart_deployment(deployment)
+
+    return get_csai_config_data(user=user, k8s_object=k8s_object, settings=settings)

pyxecm/customizer/api/v1_maintenance/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_maintenance/functions.py

@@ -0,0 +1,100 @@
+"""Define functions for v1_maintenance."""
+
+import logging
+import os
+from http import HTTPStatus
+
+import yaml
+from fastapi import HTTPException
+from pydantic import HttpUrl, ValidationError
+
+from pyxecm.customizer.api.v1_maintenance.models import MaintenanceModel
+from pyxecm.customizer.k8s import K8s
+from pyxecm.maintenance_page.settings import settings as maint_settings
+
+logger = logging.getLogger("v1_maintenance.functions")
+
+
+def get_cshost(k8s_object: K8s) -> str:
+    """Get the cs_hostname from the environment Variable OTCS_PUBLIC_HOST otherwise read it from the otcs-frontend-configmap."""
+
+    if "OTCS_PUBLIC_URL" in os.environ:
+        return os.getenv("OTCS_PUBLIC_URL", "otcs")
+
+    else:
+        cm = k8s_object.get_config_map("otcs-frontend-configmap")
+
+        if cm is None:
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail=f"Could not read otcs-frontend-configmap from namespace: {k8s_object.get_namespace()}",
+            )
+
+        config_file = cm.data.get("config.yaml")
+        config = yaml.safe_load(config_file)
+
+        try:
+            cs_url = HttpUrl(config.get("csurl"))
+        except ValidationError as ve:
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail="Could not read otcs_host from environment variable OTCS_PULIBC_URL or configmap otcs-frontend-configmap/config.yaml/cs_url",
+            ) from ve
+        return cs_url.host
+
+
+def get_maintenance_mode_status(k8s_object: K8s) -> dict:
+    """Get status of maintenance mode.
+
+    Returns:
+        dict:
+            Details of maintenance mode.
+
+    """
+    ingress = k8s_object.get_ingress("otxecm-ingress")
+
+    if ingress is None:
+        raise HTTPException(
+            status_code=500,
+            detail="No ingress object found to read Maintenance Mode status",
+        )
+
+    enabled = False
+    for rule in ingress.spec.rules:
+        if rule.host == get_cshost(k8s_object):
+            enabled = rule.http.paths[0].backend.service.name != "otcs-frontend"
+
+    return MaintenanceModel(
+        enabled=enabled, title=maint_settings.title, text=maint_settings.text, footer=maint_settings.footer
+    )
+
+    # return {
+    #     "enabled": enabled,
+    #     "title": maint_settings.title,
+    #     "text": maint_settings.text,
+    #     "footer": maint_settings.footer,
+    # }
+
+
+def set_maintenance_mode_via_ingress(enabled: bool, k8s_object: K8s) -> None:
+    """Set maintenance mode."""
+
+    logger.warning(
+        "Setting Maintenance Mode to -> %s",
+        (enabled),
+    )
+
+    if enabled:
+        k8s_object.update_ingress_backend_services(
+            "otxecm-ingress",
+            get_cshost(k8s_object=k8s_object),
+            "customizer",
+            5555,
+        )
+    else:
+        k8s_object.update_ingress_backend_services(
+            "otxecm-ingress",
+            get_cshost(k8s_object=k8s_object),
+            "otcs-frontend",
+            80,
+        )