pyxecm 2.0.2__py3-none-any.whl → 2.0.3__py3-none-any.whl

pyxecm/customizer/settings.py

@@ -200,7 +200,7 @@ class CustomizerSettingsOTPD(BaseModel):
         description="Username of the API user to configure OTPD",
         validation_alias=AliasChoices("username", "user"),
     )
-    password: str = Field(default="", description="Password of the API user to configure OTPD")
+    password: SecretStr = Field(default="", description="Password of the API user to configure OTPD")
     url: HttpUrl | None = Field(default=None, description="URL of the OTPD service")
 
     db_importfile: str = Field(default="", description="Path to the OTPD import file")
@@ -303,7 +303,7 @@ class CustomizerSettingsM365(BaseModel):
         description="Username of the M365 tenant Admin.",
         validation_alias=AliasChoices("username", "user"),
     )
-    password: str = Field(default="", description="Password of the M365 tenant Admin.")
+    password: SecretStr = Field(default="", description="Password of the M365 tenant Admin.")
     enabled: bool = Field(default=False, description="Enable/Disable the Microsoft 365 integration.")
     tenant_id: str = Field(default="", description="TennantID of the Microsoft 365 tenant")
     client_id: str = Field(default="", description="Client ID for the Microsoft 365 tenant.")
@@ -345,6 +345,23 @@ class CustomizerSettingsAviator(BaseModel):
     """Class for Aviator related settings."""
 
     enabled: bool = Field(default=False, description="Content Aviator enabled")
+    oauth_client: str = Field(default="", description="OAuth Client ID for Content Aviator")
+    oauth_secret: str = Field(default="", description="OAuth Client Secret for Content Aviator")
+    chat_svc_url: HttpUrl = Field(
+        default="http://csai-chat-svc:3000", description="Chat Service URL for Content Aviator"
+    )
+    embed_svc_url: HttpUrl = Field(
+        default="http://csai-embed-svc:3000", description="Embed Service URL for Content Aviator"
+    )
+
+
+class CustomizerSettingsKnowledgeDiscovery(BaseModel):
+    """Class for Knowledge Discovery related settings."""
+
+    enabled: bool = Field(default=False, description="Knowledge Discovery enabled")
+    url: HttpUrl | None = Field(default=None, description="URL of the Nifi Server")
+    username: str = Field(default="admin", description="Admin username for Knowledge Dicovery (Nifi)")
+    password: SecretStr = Field(default="", description="Admin password for Knowledge Discovery (Nifi)")
 
 
 class CustomizerSettingsAVTS(BaseModel):
@@ -352,7 +369,7 @@ class CustomizerSettingsAVTS(BaseModel):
 
     enabled: bool = Field(default=False, description="Enable Aviator Search configuration")
     username: str = Field(default="", description="Admin username for Aviator Search")
-    password: str = Field(default="", description="Admin password for Aviator Search")
+    password: SecretStr = Field(default="", description="Admin password for Aviator Search")
     client_id: str = Field(default="", description="OTDS Client ID for Aviator Search")
     client_secret: str = Field(default="", description="OTDS Client Secret for Aviator Search")
     base_url: HttpUrl | None = Field(
@@ -419,6 +436,7 @@ class Settings(BaseSettings):
     m365: CustomizerSettingsM365 = CustomizerSettingsM365()
     coreshare: CustomizerSettingsCoreShare = CustomizerSettingsCoreShare()
     aviator: CustomizerSettingsAviator = CustomizerSettingsAviator()
+    otkd: CustomizerSettingsKnowledgeDiscovery = CustomizerSettingsKnowledgeDiscovery()
     avts: CustomizerSettingsAVTS = CustomizerSettingsAVTS()
     otmm: CustomizerSettingsOTMM = CustomizerSettingsOTMM()
 

pyxecm/helper/xml.py

@@ -17,7 +17,7 @@ from queue import Queue
 
 import xmltodict
 
-# we need lxml instead of stadard xml.etree to have xpath capabilities!
+# we need lxml instead of standard xml.etree to have xpath capabilities!
 from lxml import etree
 from lxml.etree import Element
 

pyxecm/otawp.py

@@ -56,7 +56,7 @@ REQUEST_HEADERS_JSON = {
 REQUEST_TIMEOUT = 120
 REQUEST_MAX_RETRIES = 10
 REQUEST_RETRY_DELAY = 30
-SYNC_PUBLISH_REQUEST_TIMEOUT = 300
+SYNC_PUBLISH_REQUEST_TIMEOUT = 600
 
 default_logger = logging.getLogger(MODULE_NAME)
 
@@ -1642,15 +1642,19 @@ class OTAWP:
                 continue
 
             if SOAP_FAULT_INDICATOR in response.text:
-                self.logger.error(
-                    "Workspace synchronization failed with error -> '%s' when calling -> %s!",
+                self.logger.warning(
+                    "Workspace synchronization failed with error -> '%s' when calling -> %s! Retry in %d seconds...",
                     self.get_soap_element(soap_response=response.text, soap_tag="faultstring"),
                     self.get_soap_element(soap_response=response.text, soap_tag="faultactor"),
+                    REQUEST_RETRY_DELAY,
                 )
                 self.logger.debug("SOAP message -> %s", response.text)
-                return None
-
-            self.logger.error("Unexpected error during workspace synchronization -> %s", response.text)
+            else:
+                self.logger.warning(
+                    "Unexpected error during workspace synchronization -> %s. Retry in %d seconds...",
+                    response.text,
+                    REQUEST_RETRY_DELAY,
+                )
             time.sleep(REQUEST_RETRY_DELAY)
             retries += 1
 

pyxecm/otca.py

@@ -1,4 +1,9 @@
-"""OTCA stands for Content Aviator and is an OpenText offering for LLMM-based Agentic AI."""
+"""OTCA stands for Content Aviator and is an OpenText offering for LLMM-based Agentic AI.
+
+The REST API is documented here (OT internal):
+https://confluence.opentext.com/display/CSAI/LLM+Project+REST+APIs
+
+"""
 
 __author__ = "Dr. Marc Diefenbruch"
 __copyright__ = "Copyright (C) 2024-2025, OpenText"
@@ -6,6 +11,7 @@ __credits__ = ["Kai-Philip Gatzweiler"]
 __maintainer__ = "Dr. Marc Diefenbruch"
 __email__ = "mdiefenb@opentext.com"
 
+import hashlib
 import json
 import logging
 import platform
@@ -47,11 +53,16 @@ class OTCA:
 
     _config: dict
     _context = ""
+    _embed_token: str | None = None
+    _chat_token: str | None = None
 
     def __init__(
         self,
         chat_url: str,
         embed_url: str,
+        otds_url: str,
+        client_id: str,
+        client_secret: str,
         otcs_object: OTCS,
         synonyms: list | None = None,
         inline_citation: bool = True,
@@ -64,6 +75,12 @@ class OTCA:
                 The Content Aviator base URL for chat.
             embed_url (str):
                 The Content Aviator base URL for embedding.
+            otds_url (str):
+                The OTDS URL.
+            client_id (str):
+                The Core Share Client ID.
+            client_secret (str):
+                The Core Share client secret.
             otcs_object (OTCS):
                 The OTCS object.
             synonyms (list):
@@ -85,12 +102,15 @@ class OTCA:
         otca_config["chatUrl"] = chat_url + "/v1/chat"
         otca_config["searchUrl"] = chat_url + "/v1/context"
         otca_config["embedUrl"] = embed_url + "/v1/embeddings"
+        otca_config["clientId"] = client_id
+        otca_config["clientSecret"] = client_secret
+        otca_config["otdsUrl"] = otds_url
 
         otca_config["synonyms"] = synonyms if synonyms else []
         otca_config["inlineCitation"] = inline_citation
 
         self._config = otca_config
-        self._access_token = otcs_object.otcs_ticket()
+        self.otcs_object = otcs_object
 
     # end method definition
 
@@ -148,12 +168,16 @@ class OTCA:
 
     # end method definition
 
-    def request_header(self, content_type: str = "") -> dict:
+    def request_header(self, service_type: str = "chat", content_type: str = "application/json") -> dict:
         """Return the request header used for requests.
 
         Consists of Bearer access token and Content Type
 
         Args:
+            service_type (str, optional):
+                Service type for which the header should be returned.
+                Either "chat" or "embed". "chat" is the default.
+
             content_type (str, optional):
                 Custom content type for the request.
                 Typical values:
@@ -172,8 +196,11 @@ class OTCA:
         if content_type:
             request_header["Content-Type"] = content_type
 
-        if self._access_token is not None:
-            request_header["Authorization"] = f"Bearer {self._access_token}"
+        if service_type == "chat" and self._chat_token is not None:
+            request_header["Authorization"] = "Bearer {}".format(self._chat_token)
+
+        elif service_type == "embed" and self._embed_token is not None:
+            request_header["Authorization"] = "Bearer {}".format(self._embed_token)
 
         return request_header
 
@@ -251,7 +278,7 @@ class OTCA:
                 # Check if Session has expired - then re-authenticate and try once more
                 elif response.status_code == 401 and retries == 0:
                     self.logger.debug("Session has expired - try to re-authenticate...")
-                    self.authenticate()
+                    self.authenticate_chat()
                     retries += 1
                 else:
                     # Handle plain HTML responses to not pollute the logs
@@ -377,8 +404,70 @@ class OTCA:
 
     # end method definition
 
+    def authenticate_chat(self) -> str | None:
+        """Authenticate for Chat service at Content Aviator / CSAI.
+
+        Returns:
+            str | None:
+                Authentication token or None if the authentication fails.
+
+        """
+
+        token = self.otcs_object.otcs_ticket() or self.otcs_object.authenticate()
+
+        if token and "otcsticket" in token:
+            # Encode the input string before hashing
+            encoded_string = token["otcsticket"].encode("utf-8")
+
+            # Create a new SHA-512 hash object
+            sha512 = hashlib.sha512()
+
+            # Update the hash object with the input string
+            sha512.update(encoded_string)
+
+            # Get the hexadecimal representation of the hash
+            hashed_output = sha512.hexdigest()
+
+            self._chat_token = hashed_output
+
+            return self._chat_token
+
+        return None
+
+    # end method definition
+
+    def authenticate_embed(self) -> str | None:
+        """Authenticate as embedding service at Content Aviator / CSAI.
+
+        Returns:
+            str | None:
+                Authentication token or None if the authentication fails.
+
+        """
+
+        url = self.config()["otdsUrl"] + "/otdsws/login"
+
+        data = {
+            "grant_type": "client_credentials",
+            "client_id": self.config()["clientId"],
+            "client_secret": self.config()["clientSecret"],
+        }
+
+        result = self.do_request(url=url, method="Post", data=data)
+
+        if result:
+            self._embed_token = result["access_token"]
+            return self._embed_token
+        else:
+            self.logger.error(
+                "Authentication failed with client ID -> '%s' against -> %s", self.config()["clientId"], url
+            )
+            return None
+
+    # end method definition
+
     def chat(self, context: str | None, messages: list, where: list) -> dict:
-        """Process a chat interaction with Content Aviator.
+        r"""Process a chat interaction with Content Aviator.
 
         Chat requests are meant to be called as end-users.  This should involve
         passing the end-user's access token via the Authorization HTTP header.
@@ -396,7 +485,82 @@ class OTCA:
                 Values need to match those passed as metadata to the embeddings API.
 
         Returns:
-            dict: _description_
+            dict: Conversation status
+
+        Example:
+        {
+            'result': 'I do not know.',
+            'called': [
+                {
+                    'name': 'breakdown_query',
+                    'arguments': {},
+                    'result': '```json{"input": ["Tell me about the calibration equipment"]}```',
+                    'showInContext': False
+                },
+                {
+                    'name': 'store_subqueries',
+                    'arguments': {
+                        '0': 'Tell me about the calibration equipment'
+                    },
+                    'showInContext': False
+                },
+                {
+                    'name': 'get_next_subquery_and_reset_segment',
+                    'arguments': {},
+                    'result': 'Tell me about the calibration equipment',
+                    'showInContext': False
+                },
+                {
+                    'name': 'segmented_query',
+                    'arguments': {},
+                    'result': 'runQuery',
+                    'showInContext': False
+                },
+                {
+                    'name': 'get_context',
+                    'arguments': {
+                        'query': 'Tell me about the calibration equipment'
+                    },
+                    'result': '',
+                    'showInContext': True
+                },
+                {
+                    'name': 'check_answer',
+                    'arguments': {},
+                    'result': 'noAnswer',
+                    'showInContext': False
+                },
+                {
+                    'name': 'segmented_query',
+                    'arguments': {},
+                    'result': 'answer',
+                    'showInContext': False
+                },
+                {
+                    'name': 'get_next_subquery_and_reset_segment',
+                    'arguments': {},
+                    'showInContext': False
+                },
+                {
+                    'name': 'general_prompt',
+                    'arguments': {...},
+                    'result': 'I do not know.',
+                    'showInContext': False
+                },
+                {
+                    'name': 'filter_references',
+                    'arguments': {},
+                    'result': '[]',
+                    'showInContext': False
+                }
+            ],
+            'references': [],
+            'context': 'Tool "get_context" called with arguments {"query":"Tell me about the calibration equipment"} and returned:',
+            'queryMetadata': {
+                'originalQuery': 'Tell me about the calibration equipment',
+                'usedQuery': 'Tell me about the calibration equipment'
+            }
+        }
 
         """
 
@@ -407,22 +571,24 @@ class OTCA:
             "context": context,
             "messages": messages,
             "where": where,
-            "synonyms": self.config()["synonyms"],
-            "inlineCitation": self.config()["inlineCitation"],
+            # "synonyms": self.config()["synonyms"],
+            # "inlineCitation": self.config()["inlineCitation"],
         }
 
         return self.do_request(
             url=request_url,
             method="POST",
             headers=request_header,
-            data=chat_data,
+            json_data=chat_data,
             timeout=None,
             failure_message="Failed to chat with Content Aviator",
         )
 
     # end method definition
 
-    def search(self, query: str, threshold: float, num_results: int, document_ids: list, workspace_ids: list) -> dict:
+    def search(
+        self, query: str, document_ids: list, workspace_ids: list, threshold: float = 0.5, num_results: int = 10
+    ) -> dict:
         """Semantic search for text chunks.
 
         Search requests are meant to be called as end-users.  This should involve
@@ -432,6 +598,10 @@ class OTCA:
         Args:
             query (str):
                 The query.
+            document_ids (list):
+                List of documents (IDs) to use as scope for the query.
+            workspace_ids (list):
+                List of workspaces (IDs) to use as scope for the query.
             threshold (float):
                 Minimum similarity score to accept a document. A value like 0.7 means
                 only bring back documents that are at least 70% similar.
@@ -439,10 +609,6 @@ class OTCA:
                 Also called "top-k". Defined how many "most similar" documents to retrieve.
                 Typical value: 3-20. Higher values gets broader context but risks pulling
                 in less relevant documents.
-            document_ids (list):
-                List of documents (IDs) to use as scope for the query.
-            workspace_ids (list):
-                List of workspaces (IDs) to use as scope for the query.
 
         Returns:
             dict:
@@ -487,9 +653,9 @@ class OTCA:
         }
 
         for document_id in document_ids:
-            search_data["metadata"].append({"documentID": document_id})
+            search_data["metadata"].append({"documentID": str(document_id)})
         for workspace_id in workspace_ids:
-            search_data["metadata"].append({"workspaceID": workspace_id})
+            search_data["metadata"].append({"workspaceID": str(workspace_id)})
 
         return self.do_request(
             url=request_url,
@@ -497,7 +663,7 @@ class OTCA:
             headers=request_header,
             data=search_data,
             timeout=None,
-            failure_message="Failed to to do a semantic search with query -> '{}'!".format(query),
+            failure_message="Failed to to do a semantic search with query -> '{}'".format(query),
         )
 
     # end method definition
@@ -541,7 +707,7 @@ class OTCA:
             return None
 
         request_url = self.config()["embedUrl"]
-        request_header = self.request_header()
+        request_header = self.request_header(service_type="embed")
 
         metadata = {}
         if workspace_id:
@@ -561,7 +727,7 @@ class OTCA:
             url=request_url,
             method="POST",
             headers=request_header,
-            data=embed_data,
+            json_data=embed_data,
             timeout=None,
             failure_message="Failed to embed content",
         )