pyxecm 3.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

pyxecm_api/app.py

@@ -8,7 +8,6 @@ __email__ = "mdiefenb@opentext.com"
 
 import logging
 import os
-import threading
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime
@@ -23,9 +22,6 @@ from prometheus_fastapi_instrumentator import Instrumentator
 from pyxecm.helper.otel_config import tracer
 from pyxecm_maintenance_page import run_maintenance_page
 
-from .agents.app import agent_routers
-from .agents.functions import register_all
-from .agents.otcm_knowledgegraph.functions import build_graph
 from .auth.router import router as auth_router
 from .common.functions import PAYLOAD_LIST
 from .common.metrics import payload_logs_by_payload, payload_logs_total
@@ -50,31 +46,20 @@ if os.path.isfile(os.path.join(api_settings.logfolder, api_settings.logfile)):
 elif not os.path.exists(api_settings.logfolder):
     os.makedirs(api_settings.logfolder)
 
-handlers = [logging.FileHandler(os.path.join(api_settings.logfolder, api_settings.logfile))]
-if api_settings.log_payload_processing:
-    handlers.append(logging.StreamHandler())
-
-logging.basicConfig(
-    format="%(asctime)s %(levelname)s [%(name)s] [%(threadName)s] %(message)s",
-    datefmt="%d-%b-%Y %H:%M:%S",
-    level=api_settings.loglevel,
-    handlers=handlers,
-)
-
 
 @asynccontextmanager
 async def lifespan(
     app: FastAPI,  # noqa: ARG001
 ) -> AsyncGenerator:
-    """Lifespan Method for FASTAPI to handle the startup and shutdown process.
+    """Lifespan function for FASTAPI to handle the startup and shutdown process.
 
     Args:
         app (FastAPI):
-            The application.
+            The FastAPI application.
 
     """
 
-    logger.debug("Settings -> %s", api_settings)
+    logger.debug("API settings -> %s", api_settings)
 
     with tracer.start_as_current_span("import_payloads"):
         if api_settings.import_payload:
@@ -89,20 +74,18 @@ async def lifespan(
             # Optional Payload
             import_payload(payload_dir=api_settings.payload_dir_optional)
 
-    logger.info("Starting maintenance_page thread...")
+    logger.info("Starting maintenance page thread...")
     if api_settings.maintenance_page:
         run_maintenance_page()
 
-    if api_settings.csai_studio_integration:
-        logger.info("Registering Content Aviator tools...")
-        register_all()
-        threading.Thread(name="KnowledgeGraph", target=build_graph).start()
-
     yield
+
     logger.info("Shutdown")
     PAYLOAD_LIST.stop_payload_processing()
 
 
+# end function lifespan
+
 app = FastAPI(
     docs_url="/api",
     title=api_settings.title,
@@ -114,7 +97,7 @@ app = FastAPI(
     openapi_tags=[
         {
             "name": "auth",
-            "description": "Authentication Endpoint - Users are authenticated against Opentext Directory Services",
+            "description": "Authentication Endpoint - Users are authenticated against OpenText Directory Services",
         },
         {
             "name": "payload",
@@ -156,8 +139,6 @@ if api_settings.csai:
     )
 
     app.include_router(router=v1_csai_router)
-    for agent_router in agent_routers:
-        app.include_router(prefix="/agents", router=agent_router)
 
 
 ## Add Prometheus Instrumentator for /metrics,
@@ -170,7 +151,7 @@ if api_settings.metrics:
 
 ## Start the API Server
 def run_api() -> None:
-    """Start the FASTAPI Webserver."""
+    """Start the FastAPI Webserver."""
 
     # Check if Temp and Log dir exists
     if not os.path.exists(api_settings.temp_dir):
@@ -183,27 +164,56 @@ def run_api() -> None:
         customizer_start_time = datetime.now(UTC).strftime(
             "%Y-%m-%d_%H-%M",
         )
-        api_settings.logfile = f"customizer_{customizer_start_time}.log"
+        api_settings.logfile = "customizer_{}.log".format(customizer_start_time)
 
     # Configure Logging for uvicorn
     log_config = uvicorn.config.LOGGING_CONFIG
 
     # Stdout
-    log_config["formatters"]["pyxecm"] = {
+    log_config["formatters"]["standard"] = {
         "()": "uvicorn.logging.DefaultFormatter",
         "fmt": "%(levelprefix)s [%(name)s] [%(threadName)s] %(message)s",
         "use_colors": True,
     }
-    log_config["handlers"]["pyxecm"] = {
-        "formatter": "pyxecm",
+
+    log_config["formatters"]["logfile"] = {
+        "()": "uvicorn.logging.DefaultFormatter",
+        "fmt": "%(asctime)s %(levelname)s [%(name)s] [%(threadName)s] %(message)s",
+        "datefmt": "%d-%b-%Y %H:%M:%S",
+        "use_colors": True,
+    }
+
+    log_config["formatters"]["accesslog"] = {
+        "()": "uvicorn.logging.AccessFormatter",
+        "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s',
+        "use_colors": False,
+    }
+
+    log_config["handlers"]["console"] = {
+        "formatter": "standard",
         "class": "logging.StreamHandler",
         "stream": "ext://sys.stdout",
     }
 
-    log_config["loggers"]["pyxecm"] = {
-        "handlers": ["pyxecm"],
+    log_config["handlers"]["file"] = {
+        "formatter": "logfile",
+        "class": "logging.FileHandler",
+        "filename": os.path.join(api_settings.logfolder, api_settings.logfile),
+        "mode": "a",
+    }
+
+    log_config["handlers"]["accesslog"] = {
+        "formatter": "accesslog",
+        "class": "logging.FileHandler",
+        "filename": os.path.join(api_settings.logfolder, "access.log"),
+        "mode": "a",
+    }
+
+    log_config["loggers"]["uvicorn.access"]["handlers"] = ["access", "accesslog"]
+
+    log_config["loggers"]["root"] = {
         "level": api_settings.loglevel,
-        "propagate": False,
+        "handlers": ["console", "file"],
     }
 
     logger.info("Starting processing thread...")

pyxecm_api/auth/functions.py

@@ -27,8 +27,8 @@ def get_groups(response: dict, token: str) -> list:
     """Get the groups of the user.
 
     Args:
-        response (_type_): _description_
-        token (_type_): _description_
+        response (dict): _description_
+        token (str): _description_
 
     Returns:
         list: _description_

pyxecm_api/auth/router.py

@@ -64,9 +64,8 @@ async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]) -> J
 async def read_users_me(current_user: Annotated[User, Depends(get_current_user)]) -> JSONResponse:
     """Get the current user.
 
-    current_user:
-        type: User
-        description: The current user.
+    current_user (User):
+        The current user.
 
     """
 

pyxecm_api/common/functions.py

@@ -2,11 +2,15 @@
 
 import logging
 import os
+import time
+from datetime import UTC, datetime
 from typing import Annotated
 
 from fastapi import Depends
+from pyxecm.otca import OTCA
 from pyxecm.otcs import OTCS
 from pyxecm_customizer import K8s, PayloadList, Settings
+from pyxecm_customizer.knowledge_graph import KnowledgeGraph
 
 from pyxecm_api.auth.functions import get_otcsticket
 from pyxecm_api.settings import CustomizerAPISettings, api_settings
@@ -18,6 +22,100 @@ LOGS_LOCK = {}
 # Initialize the globel Payloadlist object
 PAYLOAD_LIST = PayloadList(logger=logger)
 
+# This object is initialized in the build_graph() function below.
+KNOWLEDGEGRAPH_OBJECT: KnowledgeGraph = None
+
+# The following ontology is fed into the knowledge graph tool description.
+# This is currently hard-coded. Ideally this should be derived from OTCM
+# or provided via a payload file:
+
+KNOWLEDGEGRAPH_ONTOLOGY = {
+    ("Vendor", "Material", "child"): ["offers", "supplies", "provides"],
+    ("Vendor", "Purchase Order", "child"): ["supplies", "provides"],
+    ("Vendor", "Purchase Contract", "child"): ["signs", "owns"],
+    ("Material", "Vendor", "parent"): ["is supplied by"],
+    ("Purchase Order", "Material", "child"): ["includes", "is part of"],
+    ("Customer", "Sales Order", "child"): ["has ordered"],
+    ("Customer", "Sales Contract", "child"): ["signs", "owns"],
+    ("Sales Order", "Customer", "parent"): ["belongs to", "is initiated by"],
+    ("Sales Order", "Material", "child"): ["includes", "consists of"],
+    ("Sales Order", "Delivery", "child"): ["triggers", "is followed by"],
+    ("Sales Order", "Production Order", "child"): ["triggers", "is followed by"],
+    ("Sales Contract", "Material", "child"): ["includes", "consists of"],
+    ("Production Order", "Material", "child"): ["includes", "consists of"],
+    ("Production Order", "Delivery", "child"): ["triggers", "is followed by"],
+    ("Production Order", "Goods Movement", "child"): ["triggers", "is followed by"],
+    ("Delivery", "Goods Movement", "child"): ["triggers", "is followed by"],
+    ("Delivery", "Material", "child"): ["triggers", "is followed by"],
+}
+
+
+### Functions
+
+
+def get_ontology() -> dict:
+    """Get the ontology for the knowledge graph.
+
+    Returns:
+        dict: The ontology as a dictionary.
+
+    """
+
+    return KNOWLEDGEGRAPH_ONTOLOGY
+
+
+def get_knowledgegraph_object() -> KnowledgeGraph:
+    """Get the Knowledge Graph object."""
+
+    global KNOWLEDGEGRAPH_OBJECT  # noqa: PLW0603
+
+    if KNOWLEDGEGRAPH_OBJECT is None:
+        KNOWLEDGEGRAPH_OBJECT = KnowledgeGraph(otcs_object=get_otcs_object(), ontology=KNOWLEDGEGRAPH_ONTOLOGY)
+
+    return KNOWLEDGEGRAPH_OBJECT
+
+
+def build_graph() -> None:
+    """Build the knowledge Graph. And keep it updated every hour."""
+
+    def build() -> None:
+        """Build the knowledge graph once."""
+
+        logger.info("Starting knowledge graph build...")
+        start_time = datetime.now(UTC)
+        result = get_knowledgegraph_object().build_graph(
+            workspace_type_exclusions=None,
+            workspace_type_inclusions=[
+                "Vendor",
+                "Purchase Contract",
+                "Purchase Order",
+                "Material",
+                "Customer",
+                "Sales Order",
+                "Sales Contract",
+                "Delivery",
+                "Goods Movement",
+            ],
+            workers=20,  # for multi-threaded traversal
+            filter_at_traversal=True,  # also filter for workspace types if following relationships
+            relationship_types=["child"],  # only go from parent to child
+            strategy="BFS",  # Breadth-First-Search
+            metadata=True,  # don't include workspace metadata
+        )
+        end_time = datetime.now(UTC)
+        logger.info(
+            "Knowledge graph completed in %s. Processed %d workspace nodes and traversed %d workspace relationships.",
+            str(end_time - start_time),
+            result["processed"],
+            result["traversed"],
+        )
+
+    # Endless loop to build knowledge graph and update it every hour:
+    while True:
+        build()
+        logger.info("Waiting for 1 hour before rebuilding the knowledge graph...")
+        time.sleep(3600)
+
 
 def get_k8s_object() -> K8s:
     """Get an instance of a K8s object.
@@ -31,12 +129,14 @@ def get_k8s_object() -> K8s:
 
 
 def get_otcs_object() -> OTCS:
-    """Get an instance of a K8s object.
+    """Get an instance of a Content Server (OTCS) object.
 
     Returns:
-        K8s: Return a K8s object
+        OTCS:
+            Return a new OTCS object.
 
     """
+
     settings = Settings()
 
     otcs = OTCS(
@@ -52,27 +152,34 @@ def get_otcs_object() -> OTCS:
         support_path=settings.otcs.support_path,
         download_dir=settings.otcs.download_dir,
         feme_uri=settings.otcs.feme_uri,
-        logger=logger.getChild("otcs"),
+        logger=logger,
     )
 
+    # Authenticate at Content Server:
     otcs.authenticate()
 
     return otcs
 
 
-def get_otcs_object_from_otcsticket(otcsticket: Annotated[str, Depends(get_otcsticket)]) -> OTCS:
-    """Get an instance of a K8s object.
+def get_otcs_object_from_otcsticket(otcs_ticket: Annotated[str, Depends(get_otcsticket)]) -> OTCS:
+    """Get an instance of a Content Server (OTCS) object.
 
     Returns:
-        K8s: Return a K8s object
+        OTCS:
+            Return an OTCS object.
 
     """
+
     settings = Settings()
 
+    # Create an OTCS object without defining the username and password:
     otcs = OTCS(
-        protocol=settings.otcs.url_backend.scheme,
-        hostname=settings.otcs.url_backend.host,
-        port=settings.otcs.url_backend.port,
+        # protocol=settings.otcs.url_backend.scheme,
+        # hostname=settings.otcs.url_backend.host,
+        # port=settings.otcs.url_backend.port,
+        protocol=settings.otcs.url_frontend.scheme,
+        hostname=settings.otcs.url_frontend.host,
+        port=settings.otcs.url_frontend.port,
         public_url=str(settings.otcs.url),
         user_partition=settings.otcs.partition,
         resource_name=settings.otcs.resource_name,
@@ -80,19 +187,62 @@ def get_otcs_object_from_otcsticket(otcsticket: Annotated[str, Depends(get_otcst
         support_path=settings.otcs.support_path,
         download_dir=settings.otcs.download_dir,
         feme_uri=settings.otcs.feme_uri,
-        logger=logger.getChild("otcs"),
+        logger=logger,
     )
 
-    otcs._otcs_ticket = otcsticket  # noqa: SLF001
+    # Instead set the OTCS authentication ticket directly:
+    otcs._otcs_ticket = otcs_ticket  # noqa: SLF001
 
     return otcs
 
 
+def get_otca_object(otcs_object: OTCS | None = None) -> OTCA:
+    """Get the Content Aviator (OTCA) object.
+
+    Args:
+        otcs_object (OTCS | None, optional):
+            The Content Server (OTCS) object. Defaults to None.
+
+    Returns:
+        OTCA:
+            The new Content Aviator object.
+
+    """
+
+    settings = Settings()
+
+    # Get the Kubernetes object:
+    k8s_object = get_k8s_object()
+    content_system = {}
+    # Read the content system (e.g. OTCM) from the Kubernetes Config Map:
+    for service in ["chat", "embed"]:
+        cm = k8s_object.get_config_map(f"csai-{service}-svc")
+        if cm:
+            content_system[service] = cm.data.get("CONTENT_SYSTEM", "none")
+            logger.info("Set content system for '%s' to -> '%s'.", service, content_system[service])
+
+    # Create the Content Aviator object (OTCA class):
+    otca = OTCA(
+        chat_url=str(settings.aviator.chat_svc_url),
+        embed_url=str(settings.aviator.embed_svc_url),
+        studio_url=str(settings.aviator.studio_url),
+        otds_url=str(settings.otds.url_internal),
+        client_id=settings.aviator.oauth_client,
+        client_secret=settings.aviator.oauth_secret,
+        otcs_object=otcs_object,
+        content_system=content_system,
+        logger=logger.getChild("otca"),
+    )
+
+    return otca
+
+
 def get_settings() -> CustomizerAPISettings:
     """Get the API Settings object.
 
     Returns:
-        CustomizerPISettings: Returns the API Settings
+        CustomizerPISettings:
+            Returns the API Settings.
 
     """
 
@@ -112,6 +262,7 @@ def get_otcs_logs_lock() -> dict:
 
 def list_files_in_directory(directory: str) -> dict:
     """Recursively list files in a directory and return a nested JSON structure with URLs."""
+
     result = {}
     for root, dirs, files in os.walk(directory):
         # Sort directories and files alphabetically
@@ -128,4 +279,5 @@ def list_files_in_directory(directory: str) -> dict:
         for file in files:
             file_path = os.path.join(relative_path, file)
             current_level[file] = file_path
+
     return result

pyxecm_api/settings.py

@@ -56,9 +56,6 @@ class CustomizerAPISettings(BaseSettings):
     )
 
     loglevel: Literal["INFO", "DEBUG", "WARNING", "ERROR"] = "INFO"
-    log_payload_processing: bool = Field(
-        default=False, description="Print the customizer payload processing log messages to stdout"
-    )
     logfolder: str = Field(
         default=os.path.join(tempfile.gettempdir(), "customizer"),
         description="Logfolder for Customizer logfiles",
@@ -122,11 +119,6 @@ class CustomizerAPISettings(BaseSettings):
         description="Prefix for the CSAI",
     )
 
-    csai_studio_integration: bool = Field(
-        default=False,
-        description="Enable the CSAI Studio Integration",
-    )
-
     upload_folder: str = Field(default=os.path.join(tempfile.gettempdir(), "upload"), description="Folder for uploads")
 
     upload_key: str = Field(default=str(uuid.uuid4()), description="Upload key for the Logs")

pyxecm_api/terminal/router.py

@@ -93,7 +93,7 @@ async def ws_terminal(
         read_task = asyncio.create_task(read_from_pty())
         write_task = asyncio.create_task(write_to_pty())
 
-        done, pending = await asyncio.wait([read_task, write_task], return_when=asyncio.FIRST_COMPLETED)
+        _done, pending = await asyncio.wait([read_task, write_task], return_when=asyncio.FIRST_COMPLETED)
 
         # Cancel other task
         for task in pending:

pyxecm_api/v1_csai/router.py

@@ -4,15 +4,14 @@ import logging
 from http import HTTPStatus
 from typing import Annotated
 
-from fastapi import APIRouter, Body, Depends, Query
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
 from fastapi.responses import HTMLResponse, JSONResponse
 from pyxecm.otcs import OTCS
 from pyxecm_customizer.k8s import K8s
 
-from pyxecm_api.agents.functions import get_otca_object
 from pyxecm_api.auth.functions import get_authorized_user
 from pyxecm_api.auth.models import User
-from pyxecm_api.common.functions import get_k8s_object, get_otcs_object, get_settings
+from pyxecm_api.common.functions import get_k8s_object, get_otca_object, get_otcs_object, get_settings
 from pyxecm_api.settings import CustomizerAPISettings
 
 from .models import CSAIEmbedMetadata
@@ -31,12 +30,16 @@ def embed_metadata(
     """Embed the Metadata of the given objects.
 
     Args:
-        user (Annotated[User, Depends): User required for authentication
-        otcs_object (Annotated[OTCS, Depends(get_otcs_object)]): OTCS object to interact with OTCS
-        body (Annotated[CSAIEmbedMetadata, Body): Request body
+        user (Annotated[User, Depends):
+            User required for authentication.
+        otcs_object (Annotated[OTCS, Depends(get_otcs_object)]):
+            The OTCS object to interact with OTCM (Content Server).
+        body (Annotated[CSAIEmbedMetadata, Body):
+            The request body.
 
     Returns:
-        JSONResponse: JSONResponse with success=true/false
+        JSONResponse:
+            JSONResponse with success=true/false
 
     """
 
@@ -53,7 +56,7 @@ def get_csai_config_data(
 ) -> JSONResponse:
     """Get the csai config data."""
 
-    logger.info("READ csai config data by user -> %s", user.id)
+    logger.info("Read CSAI config data by user -> %s", user.id)
 
     config_data = {}
 
@@ -79,9 +82,9 @@ def set_csai_config_data(
     k8s_object: Annotated[K8s, Depends(get_k8s_object)],
     config: Annotated[dict, Body()],
 ) -> JSONResponse:
-    """Get the csai config data."""
+    """Set the CSAI config data."""
 
-    logger.info("READ csai config data by user -> %s", user.id)
+    logger.info("Write CSAI config data by user -> %s", user.id)
 
     for config_map in config:
         if not config_map.startswith(settings.csai_prefix):
@@ -115,27 +118,39 @@ def set_csai_config_data(
 
 
 @router.get("/graph")
-def get_csai_graph(name: Annotated[str, Query(..., description="Name of the Graph")]) -> HTMLResponse:
+def get_csai_graph(name: Annotated[str, Query(..., description="Name of the graph")]) -> HTMLResponse:
     """Display the graph of the given name.
 
     Args:
-        otca (Annotated[OTCA, Depends): Generic OTCA Objec
-        name (str): name of the graph
+        name (str):
+            The name of the CSAI graph.
 
     Returns:
-        HTMLResponse: _description_
+        HTMLResponse: Visualization of the CSAI graph
 
     """
+
+    # Get the Content Aviator object:
     otca = get_otca_object(otcs_object=None)
 
+    # Get all graphs configured in Content Aviator:
     graphs = otca.get_graphs()
+    # Find the graph (LangGraph) with the given name:
     graph = [g for g in graphs if g["name"] == name]
 
-    if graph:
+    if not graph:
+        logger.error("Couldn't find graph -> '%s' for visualization!", name)
+        raise HTTPException(status_code=404, detail="Graph -> '{}' not found!".format(name))
+
+    try:
         filename = otca.visualize_graph(graph[0]["id"])
 
-    with open(filename) as f:
-        file_content = f.read()
+        with open(filename) as f:
+            file_content = f.read()
+    except Exception as e:
+        logger.error("Error visualizing graph -> '%s': %s", name, str(e))
+        raise HTTPException(status_code=500, detail="Failed to visualize graph -> '{}'!".format(name)) from e
+
+    logger.info("Successfully visualized graph -> '%s'", name)
 
-    logger.info("name: %s", name)
     return HTMLResponse(status_code=200, content=file_content)