pyxecm 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

pyxecm/customizer/api/v1_maintenance/models.py

@@ -0,0 +1,12 @@
+"""Define Models for the Maintenance Page Config."""
+
+from pydantic import BaseModel
+
+
+class MaintenanceModel(BaseModel):
+    """Status object of the Maintenance Page."""
+
+    enabled: bool
+    title: str | None = ""
+    text: str | None = ""
+    footer: str | None = ""

pyxecm/customizer/api/v1_maintenance/router.py

@@ -0,0 +1,76 @@
+"""Define router for v1_maintenance."""
+
+import logging
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, Form
+
+from pyxecm.customizer.api.auth import models
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.common.functions import get_k8s_object
+from pyxecm.customizer.api.v1_maintenance.functions import get_maintenance_mode_status, set_maintenance_mode_via_ingress
+from pyxecm.customizer.api.v1_maintenance.models import MaintenanceModel
+from pyxecm.customizer.k8s import K8s
+from pyxecm.maintenance_page.settings import settings as maint_settings
+
+router = APIRouter(prefix="/api/v1/maintenance", tags=["maintenance"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_maintenance")
+
+
+@router.get(path="")
+async def status(
+    user: Annotated[models.User, Depends(get_authorized_user)],  # noqa: ARG001
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+) -> MaintenanceModel:
+    """Return status of maintenance mode.
+
+    Args:
+        user (models.User):
+            Added to enforce authentication requirement
+        k8s_object (K8s):
+            K8s object instance of pyxecm K8s class
+
+
+    Returns:
+        dict:
+            Details of maintenance mode.
+
+    """
+
+    return get_maintenance_mode_status(k8s_object)
+
+
+@router.post(path="")
+async def set_maintenance_mode_options(
+    user: Annotated[models.User, Depends(get_authorized_user)],  # noqa: ARG001
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    config: Annotated[MaintenanceModel, Form()],
+) -> MaintenanceModel:
+    """Configure the Maintenance Mode and set options.
+
+    Args:
+        user (models.User):
+            Added to enforce authentication requirement
+        k8s_object (K8s):
+            K8s object instance of pyxecm K8s class
+        config (MaintenanceModel):
+            instance of the Maintenance Model
+
+    Returns:
+        dict: _description_
+
+    """
+    # Enable / Disable the acutual Maintenance Mode
+    set_maintenance_mode_via_ingress(config.enabled, k8s_object)
+
+    if config.title:
+        maint_settings.title = config.title
+
+    if config.text:
+        maint_settings.text = config.text
+
+    if config.footer:
+        maint_settings.footer = config.footer
+
+    return get_maintenance_mode_status(k8s_object)

pyxecm/customizer/api/v1_otcs/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_otcs/functions.py

@@ -0,0 +1,61 @@
+"""Define functions for v1_otcs."""
+
+import logging
+from datetime import datetime, timezone
+from threading import Lock
+
+from fastapi import APIRouter
+
+from pyxecm.customizer.api.settings import CustomizerAPISettings
+from pyxecm.customizer.k8s import K8s
+
+router = APIRouter(prefix="/api/v1/otcs", tags=["otcs"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_otcs")
+
+
+def collect_otcs_logs(host: str, k8s_object: K8s, logs_lock: Lock, settings: CustomizerAPISettings) -> None:
+    """Collect the logs for the given OTCS instance."""
+
+    with logs_lock:
+        timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d_%H-%M")
+        tgz_file = f"/tmp/{timestamp}_{host}.tar.gz"  # noqa: S108
+
+        if host.startswith("otcs-frontend"):
+            container = "otcs-frontend-container"
+        elif host.startswith("otcs-backend-search"):
+            container = "otcs-backend-search-container"
+        elif host.startswith("otcs-admin"):
+            container = "otcs-admin-container"
+        else:
+            container = None
+
+        logger.info("Collecting logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=["tar", "-czvf", tgz_file, "/opt/opentext/cs/logs", "/opt/opentext/cs_persist/contentserver.log"],
+            container=container,
+            timeout=1800,
+        )
+
+        logger.info("Uploading logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=[
+                "curl",
+                "-X",
+                "POST",
+                "-F",
+                f"file=@{tgz_file}",
+                f"{settings.upload_url}?key={settings.upload_key}",
+            ],
+            container=container,
+            timeout=1800,
+        )
+
+        logger.info("Cleanup logs for %s", host)
+        k8s_object.exec_pod_command(
+            pod_name=host,
+            command=["rm", tgz_file],
+            container=container,
+        )

pyxecm/customizer/api/v1_otcs/router.py

@@ -0,0 +1,179 @@
+"""Define router for v1_otcs."""
+
+import logging
+import os
+import shutil
+from http import HTTPStatus
+from threading import Lock, Thread
+from typing import Annotated
+
+import anyio
+from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
+from fastapi.responses import FileResponse, JSONResponse
+
+from pyxecm.customizer.api.auth.functions import get_authorized_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.common.functions import get_k8s_object, get_otcs_logs_lock, get_settings
+from pyxecm.customizer.api.settings import CustomizerAPISettings
+from pyxecm.customizer.api.v1_otcs.functions import collect_otcs_logs
+from pyxecm.customizer.k8s import K8s
+
+router = APIRouter(prefix="/api/v1/otcs", tags=["otcs"])
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_otcs")
+
+
+@router.put(path="/logs", tags=["otcs"])
+async def put_otcs_logs(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    otcs_logs_lock: Annotated[dict[str, Lock], Depends(get_otcs_logs_lock)],
+    hosts: Annotated[list[str], Body()],
+) -> JSONResponse:
+    """Collect the logs from the given OTCS instances."""
+
+    if "all" in hosts:
+        hosts = []
+        for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search"]:
+            try:
+                sts_replicas = k8s_object.get_stateful_set_scale(sts).status.replicas
+                hosts.extend([f"{sts}-{i}" for i in range(sts_replicas)])
+            except Exception as e:
+                raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) from e
+
+    msg = {}
+    for host in hosts:
+        if host not in otcs_logs_lock:
+            otcs_logs_lock[host] = Lock()
+
+        if not otcs_logs_lock[host].locked():
+            Thread(target=collect_otcs_logs, args=(host, k8s_object, otcs_logs_lock[host], settings)).start()
+            msg[host] = {"status": "ok", "message": "Logs are being collected"}
+        else:
+            msg[host] = {"status": "error", "message": "Logs are already being collected"}
+
+    status = (
+        HTTPStatus.TOO_MANY_REQUESTS if any(msg[host]["status"] == "error" for host in msg) else HTTPStatus.ACCEPTED
+    )
+    return JSONResponse(msg, status_code=status)
+
+
+@router.post("/logs/upload", tags=["otcs"], include_in_schema=True)
+async def post_otcs_log_file(
+    file: Annotated[UploadFile, File(...)],
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    key: str = "",
+) -> JSONResponse:
+    """Upload a file to disk.
+
+    Args:
+        file: File to be uploaded.
+        settings: CustomizerAPISettings.
+        key: Key to validate the upload.
+
+    Returns:
+        JSONResponse: Status of the upload
+
+    """
+    if key != settings.upload_key:
+        raise HTTPException(status_code=403, detail="Invalid Uploadkey")
+
+    os.makedirs(settings.upload_folder, exist_ok=True)
+
+    try:
+        async with await anyio.open_file(os.path.join(settings.upload_folder, file.filename), "wb") as f:
+            # Process the file in chunks instead of loading the entire file into memory
+            while True:
+                chunk = await file.read(65536)  # Read 64KB at a time
+                if not chunk:
+                    break
+                await f.write(chunk)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Something went wrong") from e
+    finally:
+        await file.close()
+
+    return {"message": f"Successfully uploaded {file.filename}"}
+
+
+@router.get("/logs", tags=["otcs"])
+async def get_otcs_log_files(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    k8s_object: Annotated[K8s, Depends(get_k8s_object)],
+    otcs_logs_lock: Annotated[dict[str, Lock], Depends(get_otcs_logs_lock)],
+) -> JSONResponse:
+    """List all otcs logs that can be downloaded."""
+
+    os.makedirs(settings.upload_folder, exist_ok=True)
+
+    files = []
+    for filename in sorted(os.listdir(settings.upload_folder)):
+        file_path = os.path.join(settings.upload_folder, filename)
+        if os.path.isfile(file_path):
+            file_size = os.path.getsize(file_path)
+            files.append({"filename": filename, "size": file_size})
+
+    response = {"status": {host: bool(otcs_logs_lock[host].locked()) for host in otcs_logs_lock}, "files": files}
+
+    # Extend response with all hosts
+    for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search"]:
+        try:
+            sts_replicas = k8s_object.get_stateful_set_scale(sts).status.replicas
+            for i in range(sts_replicas):
+                host = f"{sts}-{i}"
+
+                if host in otcs_logs_lock:
+                    response["status"][host] = otcs_logs_lock[host].locked()
+                else:
+                    response["status"][host] = False
+
+        except Exception as e:
+            raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) from e
+
+    return JSONResponse(response, status_code=HTTPStatus.OK)
+
+
+@router.delete("/logs", tags=["otcs"])
+async def delete_otcs_log_files(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+) -> JSONResponse:
+    """Delete all otcs log files."""
+    shutil.rmtree(settings.upload_folder)
+    return JSONResponse({"message": "Successfully deleted all files"}, status_code=HTTPStatus.OK)
+
+
+@router.delete("/logs/{file_name}", tags=["otcs"])
+async def delete_otcs_log_file(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    file_name: str,
+) -> FileResponse:
+    """Delete single OTCS log archive."""
+    file_path = os.path.join(settings.upload_folder, file_name)
+
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="File not found")
+
+    try:
+        os.remove(file_path)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"{e}") from e
+
+    return JSONResponse({"message": f"Successfully deleted {file_name}"}, status_code=HTTPStatus.OK)
+
+
+@router.get("/logs/{file_name}", tags=["otcs"])
+async def get_otcs_log_file(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    settings: Annotated[CustomizerAPISettings, Depends(get_settings)],
+    file_name: str,
+) -> FileResponse:
+    """Download OTCS log archive."""
+    file_path = os.path.join(settings.upload_folder, file_name)
+    if not os.path.exists(file_path):
+        raise HTTPException(status_code=404, detail="File not found")
+
+    return FileResponse(file_path, media_type="application/octet-stream", filename=file_name)

pyxecm/customizer/api/v1_payload/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/v1_payload/functions.py

@@ -0,0 +1,179 @@
+"""API Implemenation for the Customizer to start and control the payload processing."""
+
+__author__ = "Dr. Marc Diefenbruch"
+__copyright__ = "Copyright (C) 2024-2025, OpenText"
+__credits__ = ["Kai-Philip Gatzweiler"]
+__maintainer__ = "Dr. Marc Diefenbruch"
+__email__ = "mdiefenb@opentext.com"
+
+import asyncio
+import logging
+import os
+from collections.abc import AsyncGenerator
+
+import anyio
+
+from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
+from pyxecm.customizer.api.settings import api_settings
+from pyxecm.customizer.exceptions import PayloadImportError
+from pyxecm.customizer.payload import load_payload
+
+logger = logging.getLogger("pyxecm.customizer.api.v1_payload")
+
+# Initialize the globel Payloadlist object
+
+
+def import_payload(
+    payload: str | None = None,
+    payload_dir: str | None = None,
+    enabled: bool | None = None,
+    dependencies: bool | None = None,
+) -> None:
+    """Automatically load payload items from disk of a given directory.
+
+    Args:
+        payload (str):
+            The name of the payload.
+        payload_dir (str):
+            The local path.
+        enabled (bool, optional):
+            Automatically start the processing (True), or only define items (False).
+            Defaults to False.
+        dependencies (bool, optional):
+            Automatically add dependency on the last payload in the queue
+
+    """
+
+    def import_payload_file(
+        filename: str,
+        enabled: bool | None,
+        dependencies: bool | None,
+    ) -> None:
+        if not os.path.isfile(filename):
+            return
+
+        if not (filename.endswith((".yaml", ".tfvars", ".tf", ".yml.gz.b64"))):
+            logger.debug("Skipping file: %s", filename)
+            return
+
+        # Load payload file
+        payload_content = load_payload(filename)
+        if payload_content is None:
+            exception = f"The import of payload -> {filename} failed. Payload content could not be loaded."
+            raise PayloadImportError(exception)
+
+        payload_options = payload_content.get("payloadOptions", {})
+
+        if enabled is None:
+            enabled = payload_options.get("enabled", True)
+
+        # read name from options section if specified, otherwise take filename
+        name = payload_options.get("name", os.path.basename(filename))
+
+        # Get the loglevel from payloadOptions if set, otherwise use the default loglevel
+        loglevel = payload_options.get("loglevel", api_settings.loglevel)
+
+        # Get the git_url
+        git_url = payload_options.get("git_url", None)
+
+        # Dependency Management
+        if dependencies is None:
+            dependencies = []
+
+            # Get all dependencies from payloadOptions and resolve their ID
+            for dependency_name in payload_options.get("dependencies", []):
+                dependend_item = PAYLOAD_LIST.get_payload_item_by_name(dependency_name)
+
+                if dependend_item is None:
+                    exception = (
+                        f"The import of payload -> {name} failed. Dependencies cannot be resovled: {dependency_name}",
+                    )
+                    raise PayloadImportError(
+                        exception,
+                    )
+                # Add the ID to the list of dependencies
+                dependencies.append(dependend_item["index"])
+
+        elif dependencies:
+            try:
+                payload_items = len(PAYLOAD_LIST.get_payload_items()) - 1
+                dependencies = [payload_items] if payload_items != -1 else []
+            except Exception:
+                dependencies = []
+        else:
+            dependencies = []
+
+        customizer_settings = payload_content.get("customizerSettings", {})
+
+        logger.info("Adding payload: %s", filename)
+        payload = PAYLOAD_LIST.add_payload_item(
+            name=name,
+            filename=filename,
+            status="planned",
+            logfile=f"{api_settings.logfolder}/{name}.log",
+            dependencies=dependencies,
+            enabled=enabled,
+            git_url=git_url,
+            loglevel=loglevel,
+            customizer_settings=customizer_settings,
+        )
+        dependencies = payload["index"]
+
+        return
+
+    if payload is None and payload_dir is None:
+        exception = "No payload or payload_dir provided"
+        raise ValueError(exception)
+
+    if payload and os.path.isdir(payload) and payload_dir is None:
+        payload_dir = payload
+
+    if payload_dir is None:
+        import_payload_file(payload, enabled, dependencies)
+        return
+    elif not os.path.isdir(payload_dir):
+        return
+
+    for filename in sorted(os.listdir(payload_dir)):
+        try:
+            import_payload_file(os.path.join(payload_dir, filename), enabled, dependencies)
+        except PayloadImportError:
+            logger.error("Payload import failed")
+
+
+def prepare_dependencies(dependencies: list) -> list | None:
+    """Convert the dependencies string to a list of integers."""
+    try:
+        list_all = dependencies[0].split(",")
+    except IndexError:
+        return None
+
+    # Remove empty values from the list
+    items = list(filter(None, list_all))
+    converted_list = []
+    for item in items:
+        try:
+            converted_list.append(int(item))
+        except ValueError:
+            continue
+
+    return converted_list
+
+
+async def tail_log(file_path: str) -> AsyncGenerator[str, None]:
+    """Asynchronously follow the log file like `tail -f`."""
+    try:
+        async with await anyio.open_file(file_path) as file:
+            # Move the pointer to the end of the file
+            await file.seek(0, os.SEEK_END)
+
+            while True:
+                # Read new line
+                line = await file.readline()
+                if not line:
+                    # Sleep for a little while before checking for new lines
+                    await asyncio.sleep(0.5)
+                    continue
+                yield line
+    except asyncio.exceptions.CancelledError:
+        pass

pyxecm/customizer/api/v1_payload/models.py

@@ -0,0 +1,51 @@
+"""Define Models for Payload."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class PayloadStats(BaseModel):
+    """Defines PayloadStats Model."""
+
+    count: int = 0
+    status: dict = {}
+    logs: dict = {}
+
+
+class PayloadListItem(BaseModel):
+    """Defines PayloadListItem Model."""
+
+    index: int
+    name: str
+    filename: str
+    dependencies: list
+    logfile: str
+    status: str
+    enabled: bool
+    git_url: str | None
+    loglevel: str = "INFO"
+    start_time: Any | None
+    stop_time: Any | None
+    duration: Any | None
+    log_debug: int = 0
+    log_info: int = 0
+    log_warning: int = 0
+    log_error: int = 0
+    log_critical: int = 0
+    customizer_settings: dict = {}
+
+
+class PayloadListItems(BaseModel):
+    """Defines PayloadListItems Model."""
+
+    stats: PayloadStats
+    results: list[PayloadListItem]
+
+
+class UpdatedPayloadListItem(BaseModel):
+    """Defines UpdatedPayloadListItem Model."""
+
+    message: str
+    payload: PayloadListItem
+    updated_fields: dict