pyxecm 1.6__py3-none-any.whl → 2.0.1__py3-none-any.whl

pyxecm/customizer/__init__.py

@@ -1,12 +1,24 @@
-"""PYXECM classes for Customizer"""
+"""PYXECM classes for Customizer."""
 
 from .browser_automation import BrowserAutomation
-
 from .customizer import Customizer
 from .k8s import K8s
 from .m365 import M365
 from .payload import Payload
-from .sap import SAP
 from .salesforce import Salesforce
-from .successfactors import SuccessFactors
+from .sap import SAP
 from .servicenow import ServiceNow
+from .successfactors import SuccessFactors
+
+__all__ = [
+    "M365",
+    "SAP",
+    "BrowserAutomation",
+    "Customizer",
+    "Guidewire",
+    "K8s",
+    "Payload",
+    "Salesforce",
+    "ServiceNow",
+    "SuccessFactors",
+]

pyxecm/customizer/__main__.py

@@ -0,0 +1,58 @@
+"""Start the Customizer to process a payload."""
+
+__author__ = "Dr. Marc Diefenbruch"
+__copyright__ = "Copyright (C) 2024-2025, OpenText"
+__credits__ = ["Kai-Philip Gatzweiler"]
+__maintainer__ = "Dr. Marc Diefenbruch"
+__email__ = "mdiefenb@opentext.com"
+
+import logging
+import os
+import sys
+
+from pyxecm.customizer import Customizer
+from pyxecm.customizer.payload import load_payload
+
+logger = logging.getLogger("customizer")
+
+
+def main(argv: list[str]) -> int:
+    """Start the Customizer."""
+
+    if len(argv) < 2:
+        logger.error("No input file specified")
+        sys.exit(1)
+
+    payload_filename = argv[1]
+
+    if not os.path.isfile(payload_filename):
+        logger.error("Input file does not exist")
+        sys.exit(1)
+
+    payload = load_payload(payload_filename)
+    customizer_settings = payload.get("customizerSettings", {})
+
+    # Overwrite the customizer settings with the payload specific ones:
+    customizer_settings.update({"cust_payload": payload_filename})
+
+    my_customizer = Customizer(logger=logger, settings=customizer_settings)
+
+    my_customizer.customization_run()
+
+    return 0
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+        datefmt="%d-%b-%Y %H:%M:%S",
+        level=logging.INFO,
+        handlers=[
+            logging.StreamHandler(sys.stdout),
+        ],
+    )
+
+    try:
+        main(sys.argv)
+    except KeyboardInterrupt:
+        logger.warning("KeyboardInterrupt - exiting")

pyxecm/customizer/api/__init__.py

@@ -0,0 +1,5 @@
+"""PYXECM API main imports."""
+
+from .app import app, run_api
+
+__all__ = ["app", "run_api"]

pyxecm/customizer/api/__main__.py

@@ -0,0 +1,6 @@
+"""Wrapper to start the Customizer API."""
+
+from pyxecm.customizer.api import run_api
+
+if __name__ == "__main__":
+    run_api()

pyxecm/customizer/api/app.py

@@ -0,0 +1,163 @@
+"""API Implemenation for the Customizer to start and control the payload processing."""
+
+__author__ = "Dr. Marc Diefenbruch"
+__copyright__ = "Copyright (C) 2024-2025, OpenText"
+__credits__ = ["Kai-Philip Gatzweiler"]
+__maintainer__ = "Dr. Marc Diefenbruch"
+__email__ = "mdiefenb@opentext.com"
+
+import logging
+import os
+import sys
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from importlib.metadata import version
+from threading import Thread
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from prometheus_fastapi_instrumentator import Instrumentator
+
+from pyxecm.customizer.api.auth.router import router as auth_router
+from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
+from pyxecm.customizer.api.common.metrics import payload_logs_by_payload, payload_logs_total
+from pyxecm.customizer.api.common.router import router as common_router
+from pyxecm.customizer.api.settings import api_settings
+from pyxecm.customizer.api.terminal.router import router as terminal_router
+from pyxecm.customizer.api.v1_csai.router import router as v1_csai_router
+from pyxecm.customizer.api.v1_maintenance.router import router as v1_maintenance_router
+from pyxecm.customizer.api.v1_otcs.router import router as v1_otcs_router
+from pyxecm.customizer.api.v1_payload.functions import import_payload
+from pyxecm.customizer.api.v1_payload.router import router as v1_payload_router
+from pyxecm.maintenance_page import run_maintenance_page
+
+# Check if Temp dir exists
+if not os.path.exists(api_settings.temp_dir):
+    os.makedirs(api_settings.temp_dir)
+
+# Check if Logfile and folder exists and is unique
+if os.path.isfile(os.path.join(api_settings.logfolder, api_settings.logfile)):
+    customizer_start_time = datetime.now(timezone.utc).strftime(
+        "%Y-%m-%d_%H-%M",
+    )
+    api_settings.logfile = f"customizer_{customizer_start_time}.log"
+elif not os.path.exists(api_settings.logfolder):
+    os.makedirs(api_settings.logfolder)
+
+handlers = [
+    logging.FileHandler(os.path.join(api_settings.logfolder, api_settings.logfile)),
+    logging.StreamHandler(sys.stdout),
+]
+
+logging.basicConfig(
+    format="%(asctime)s %(levelname)s [%(name)s] [%(threadName)s] %(message)s",
+    datefmt="%d-%b-%Y %H:%M:%S",
+    level=api_settings.loglevel,
+    handlers=handlers,
+)
+
+
+@asynccontextmanager
+async def lifespan(
+    app: FastAPI,  # noqa: ARG001
+) -> AsyncGenerator:
+    """Lifespan Method for FASTAPI to handle the startup and shutdown process.
+
+    Args:
+        app (FastAPI):
+            The application.
+
+    """
+
+    logger.debug("Settings -> %s", api_settings)
+
+    if api_settings.import_payload:
+        logger.info("Importing filesystem payloads...")
+
+        # Base Payload
+        import_payload(payload=api_settings.payload)
+
+        # External Payload
+        import_payload(payload_dir=api_settings.payload_dir, dependencies=True)
+
+        # Optional Payload
+        import_payload(payload_dir=api_settings.payload_dir_optional)
+
+    if api_settings.maintenance_mode:
+        logger.info("Starting maintenance_page thread...")
+        maint_thread = Thread(target=run_maintenance_page, name="maintenance_page")
+        maint_thread.start()
+
+    logger.info("Starting processing thread...")
+    thread = Thread(
+        target=PAYLOAD_LIST.run_payload_processing,
+        name="customization_run_api",
+    )
+    thread.start()
+
+    yield
+    logger.info("Shutdown")
+    PAYLOAD_LIST.stop_payload_processing()
+
+
+app = FastAPI(
+    docs_url="/api",
+    title="Customizer API",
+    openapi_url="/api/openapi.json",
+    lifespan=lifespan,
+    version=version("pyxecm"),
+    openapi_tags=[
+        {
+            "name": "auth",
+            "description": "Authentication Endpoint - Users are authenticated against Opentext Directory Services",
+        },
+        {
+            "name": "payload",
+            "description": "Get status and manipulate payload objects ",
+        },
+        {
+            "name": "maintenance",
+            "description": "Enable, disable or alter the maintenance mode.",
+        },
+    ],
+)
+
+## Add all Routers
+app.include_router(router=common_router)
+app.include_router(router=auth_router)
+app.include_router(router=v1_maintenance_router)
+app.include_router(router=v1_otcs_router)
+app.include_router(router=v1_payload_router)
+if api_settings.ws_terminal:
+    app.include_router(router=terminal_router)
+if api_settings.csai:
+    app.include_router(router=v1_csai_router)
+
+
+logger = logging.getLogger("CustomizerAPI")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=api_settings.trusted_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+if api_settings.metrics:
+    # Add Prometheus Instrumentator for /metrics
+    instrumentator = Instrumentator().instrument(app).expose(app)
+    instrumentator.add(payload_logs_by_payload(PAYLOAD_LIST))
+    instrumentator.add(payload_logs_total(PAYLOAD_LIST))
+
+
+def run_api() -> None:
+    """Start the FASTAPI Webserver."""
+
+    uvicorn.run(
+        "pyxecm.customizer.api:app",
+        host=api_settings.bind_address,
+        port=api_settings.bind_port,
+        workers=api_settings.workers,
+    )

pyxecm/customizer/api/auth/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/auth/functions.py

@@ -0,0 +1,92 @@
+"""Utility library to handle the authentication with OTDS."""
+
+import json
+from typing import Annotated
+
+import requests
+from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer
+
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.settings import api_settings
+
+router = APIRouter()
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+
+
+def get_groups(response: dict, token: str) -> list:
+    """Get the groups of the user.
+
+    Args:
+        response (_type_): _description_
+        token (_type_): _description_
+
+    Returns:
+        list: _description_
+
+    """
+
+    headers = {
+        "Accept": "application/json",
+        "otdsticket": token,
+    }
+    url = api_settings.otds_url + "/otdsws/rest/users/" + response["user"]["id"] + "/memberof"
+
+    response = requests.request("GET", url, headers=headers, timeout=5)
+    if response.ok:
+        response = json.loads(response.text)
+        return [group["id"] for group in response.get("groups", [])]
+
+    # Retur empty list if request wasn't successful
+    return []
+
+
+async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]) -> User:
+    """Get the current user from OTDS and verify it."""
+
+    if api_settings.api_token is not None and token == api_settings.api_token:
+        return User(
+            id="api",
+            full_name="API Token",
+            groups=["otadmins@otds.admin"],
+            is_admin=True,
+            is_sysadmin=True,
+        )
+
+    url = api_settings.otds_url + "/otdsws/rest/currentuser"
+    headers = {
+        "Accept": "application/json",
+        "otdsticket": token,
+    }
+    response = requests.request("GET", url, headers=headers, timeout=2)
+
+    if response.ok:
+        response = json.loads(response.text)
+
+        user = User(
+            id=response["user"]["id"],
+            full_name=response["user"]["name"],
+            groups=get_groups(response, token),
+            is_admin=response["isAdmin"],
+            is_sysadmin=response["isSysAdmin"],
+        )
+
+        return user
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authentication credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+
+async def get_authorized_user(current_user: Annotated[User, Depends(get_current_user)]) -> User:
+    """Check if the user is authorized (member of the Group otadmin@otds.admin)."""
+
+    if "otadmins@otds.admin" not in current_user.groups:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=f"User {current_user.id} is not authorized",
+        )
+    return current_user

pyxecm/customizer/api/auth/models.py

@@ -0,0 +1,13 @@
+"""Models for FastAPI."""
+
+from pydantic import BaseModel
+
+
+class User(BaseModel):
+    """Model for users authenticated by OTDS."""
+
+    id: str
+    full_name: str | None = None
+    groups: list[str] | None = None
+    is_admin: bool = False
+    is_sysadmin: bool = False

pyxecm/customizer/api/auth/router.py

@@ -0,0 +1,78 @@
+"""Utility library to handle the authentication with OTDS."""
+
+import json
+from typing import Annotated
+
+import requests
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+
+from pyxecm.customizer.api.auth.functions import get_current_user
+from pyxecm.customizer.api.auth.models import User
+from pyxecm.customizer.api.settings import api_settings
+
+router = APIRouter()
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+
+
+@router.post("/token", tags=["auth"])
+async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> JSONResponse:
+    """Login using OTDS and return a token."""
+
+    url = api_settings.otds_url + "/otdsws/rest/authentication/credentials"
+
+    payload = json.dumps(
+        {"userName": form_data.username, "password": form_data.password},
+    )
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    try:
+        response = requests.request(
+            "POST",
+            url,
+            headers=headers,
+            data=payload,
+            timeout=10,
+        )
+    except requests.exceptions.ConnectionError as exc:
+        raise HTTPException(
+            status_code=500,
+            detail=f"{exc.request.url} cannot be reached",
+        ) from exc
+
+    if response.ok:
+        response = json.loads(response.text)
+    else:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+
+    return JSONResponse(
+        {
+            "access_token": response["ticket"],
+            "token_type": "bearer",
+            "userId": response["userId"],
+        },
+    )
+
+
+@router.get("/users/me", tags=["auth"])
+async def read_users_me(current_user: Annotated[User, Depends(get_current_user)]) -> JSONResponse:
+    """Get the current user.
+
+    current_user:
+        type: User
+        description: The current user.
+
+    """
+
+    if "otadmins@otds.admin" in current_user.groups:
+        return JSONResponse(current_user.model_dump())
+    else:
+        raise HTTPException(
+            status_code=403,
+            detail=f"User {current_user.id} is not authorized",
+        )

pyxecm/customizer/api/common/__init__.py

@@ -0,0 +1 @@
+"""init module."""

pyxecm/customizer/api/common/functions.py

@@ -0,0 +1,47 @@
+"""Define common functions."""
+
+import logging
+
+from pyxecm.customizer.api.common.payload_list import PayloadList
+from pyxecm.customizer.api.settings import CustomizerAPISettings, api_settings
+from pyxecm.customizer.k8s import K8s
+
+logger = logging.getLogger("pyxecm.customizer.api")
+
+# Create a LOCK dict for singleton logs collection
+LOGS_LOCK = {}
+# Initialize the globel Payloadlist object
+PAYLOAD_LIST = PayloadList(logger=logger)
+
+
+def get_k8s_object() -> K8s:
+    """Get an instance of a K8s object.
+
+    Returns:
+        K8s: Return a K8s object
+
+    """
+
+    return K8s(logger=logger, namespace=api_settings.namespace)
+
+
+def get_settings() -> CustomizerAPISettings:
+    """Get the API Settings object.
+
+    Returns:
+        CustomizerPISettings: Returns the API Settings
+
+    """
+
+    return api_settings
+
+
+def get_otcs_logs_lock() -> dict:
+    """Get the Logs LOCK dict.
+
+    Returns:
+        The dict with all LOCKS for the logs
+
+    """
+
+    return LOGS_LOCK

pyxecm/customizer/api/common/metrics.py

@@ -0,0 +1,92 @@
+"""Metics for payload logs."""
+
+from collections.abc import Callable
+
+from prometheus_client import Gauge
+from prometheus_fastapi_instrumentator.metrics import Info
+
+from pyxecm.customizer.api.common.payload_list import PayloadList
+
+
+## By Payload
+def payload_logs_by_payload(payload_list: PayloadList) -> Callable[[Info], None]:
+    """Metrics for payload logs by payload."""
+
+    metrics_error = Gauge(
+        "payload_error",
+        "Number of ERROR log messages for by payload",
+        labelnames=("index", "name", "logfile"),
+    )
+
+    metrics_warning = Gauge(
+        "payload_warning",
+        "Number of WARNING log messages for by payload",
+        labelnames=("index", "name", "logfile"),
+    )
+
+    metrics_info = Gauge(
+        "payload_info",
+        "Number of INFO log messages for by payload",
+        labelnames=("index", "name", "logfile"),
+    )
+
+    metrics_debug = Gauge(
+        "payload_debug",
+        "Number of DEBUG log messages for by payload",
+        labelnames=("index", "name", "logfile"),
+    )
+
+    def instrumentation(info: Info) -> None:  # noqa: ARG001
+        df = payload_list.get_payload_items()
+        data = [{"index": idx, **row} for idx, row in df.iterrows()]
+
+        for item in data:
+            metrics_error.labels(item["index"], item["name"], item["logfile"]).set(
+                item["log_error"],
+            )
+            metrics_warning.labels(item["index"], item["name"], item["logfile"]).set(
+                item["log_warning"],
+            )
+            metrics_info.labels(item["index"], item["name"], item["logfile"]).set(
+                item["log_info"],
+            )
+            metrics_debug.labels(item["index"], item["name"], item["logfile"]).set(
+                item["log_debug"],
+            )
+
+    return instrumentation
+
+
+## Total
+def payload_logs_total(payload_list: PayloadList) -> Callable[[Info], None]:
+    """Metrics for total payload logs messages."""
+
+    metrics_error = Gauge(
+        "payload_error_total",
+        "Total number of ERROR log messages",
+    )
+
+    metrics_warning = Gauge(
+        "payload_warning_total",
+        "Total number of WARNING log messages",
+    )
+
+    metrics_info = Gauge(
+        "payload_info_total",
+        "Total number of INFO log messages",
+    )
+
+    metrics_debug = Gauge(
+        "payload_debug_total",
+        "Total number of DEBUG log messages",
+    )
+
+    def instrumentation(info: Info) -> None:  # noqa: ARG001
+        df = payload_list.get_payload_items()
+
+        metrics_error.set(df["log_error"].sum())
+        metrics_warning.set(df["log_warning"].sum())
+        metrics_info.set(df["log_info"].sum())
+        metrics_debug.set(df["log_debug"].sum())
+
+    return instrumentation

pyxecm/customizer/api/common/models.py

@@ -0,0 +1,21 @@
+"""Define common base Models."""
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class CustomizerStatus(BaseModel):
+    """Define Model for Customizer Status."""
+
+    version: int = 2
+    customizer_duration: Any | None
+    customizer_end_time: Any | None
+    customizer_start_time: Any | None
+    status_details: dict
+    status: str = "Stopped"
+    debug: int = 0
+    info: int = 0
+    warning: int = 0
+    error: int = 0
+    critical: int = 0