pyxecm 2.0.4__py3-none-any.whl → 3.0.1__py3-none-any.whl

{pyxecm/customizer/api/common → pyxecm_customizer}/payload_list.py

@@ -15,29 +15,19 @@ import pprint
 import threading
 import time
 import traceback
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
+import pandas as pd
 from pydantic import ValidationError
-
-from pyxecm.customizer.api.settings import api_settings
+from pyxecm.helper.otel_config import trace, tracer
 
 # OpenText specific modules:
-from pyxecm.customizer.customizer import Customizer
-from pyxecm.customizer.exceptions import StopOnError
-from pyxecm.customizer.log import LogCountFilter, VictoriaLogsHandler
-from pyxecm.customizer.payload import load_payload
-
-default_logger = logging.getLogger("pyxecm.customizer.payload_list")
+from pyxecm_customizer.customizer import Customizer
+from pyxecm_customizer.exceptions import StopOnError
+from pyxecm_customizer.log import LogCountFilter
+from pyxecm_customizer.payload import load_payload
 
-try:
-    import pandas as pd
-
-    pandas_installed = True
-except ModuleNotFoundError:
-    default_logger.warning(
-        "Module pandas is not installed. Customizer will not support bulk workspace creation.",
-    )
-    pandas_installed = False
+default_logger = logging.getLogger("pyxecm_customizer.payload_list")
 
 
 class PayloadList:
@@ -93,7 +83,7 @@ class PayloadList:
 
         def calculate_duration(row: pd.Series) -> str:
             if row["status"] == "running":
-                now = datetime.now(timezone.utc)
+                now = datetime.now(UTC)
                 start_time = pd.to_datetime(row["start_time"])
 
                 duration = now - start_time
@@ -650,10 +640,14 @@ class PayloadList:
 
         """
 
+        @tracer.start_as_current_span("run_and_complete_payload")
         def run_and_complete_payload(payload_item: pd.Series) -> None:
             """Run the payload's process_payload method and marks the status as completed afterward."""
 
-            start_time = datetime.now(timezone.utc)
+            t = trace.get_current_span()
+            t.set_attributes({"payload_id": payload_item["index"], "payload_name": payload_item["name"]})
+
+            start_time = datetime.now(UTC)
             self.update_payload_item(payload_item["index"], {"start_time": start_time})
 
             # Create a logger with thread_id:
@@ -679,24 +673,6 @@ class PayloadList:
             handler.setFormatter(fmt=formatter)
             thread_logger.addHandler(hdlr=handler)
 
-            # If hostname is set, configure log handler so forward logs
-            if api_settings.victorialogs_host:
-                handler_kwargs = {
-                    "host": api_settings.victorialogs_host,
-                    "port": api_settings.victorialogs_port,
-                    "app": "Customizer",
-                    "payload_item": payload_item["index"],
-                    "payload_file": payload_item["filename"],
-                }
-
-                # Read namespace if available and add as kwarg to loghandler
-                file_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
-                if os.path.isfile(file_path):
-                    with open(file_path) as file:
-                        handler_kwargs["namespace"] = file.read()
-
-                thread_logger.addHandler(VictoriaLogsHandler(**handler_kwargs))
-
             if len(thread_logger.filters) == 0:
                 thread_logger.debug("Adding log count filter to logger")
                 thread_logger.addFilter(
@@ -749,12 +725,6 @@ class PayloadList:
                     )
 
                     if customizer_settings.get("profiling", False):
-                        from pyinstrument import Profiler
-
-                        profiler = Profiler()
-                        profiler.start()
-
-                    if customizer_settings.get("cprofiling", False):
                         import cProfile
                         import pstats
 
@@ -763,19 +733,16 @@ class PayloadList:
 
                     success = local.customizer_thread_object.customization_run()
 
-                    if customizer_settings.get("cprofiling", False):
-                        cprofiler.disable()
-
                     if customizer_settings.get("profiling", False):
-                        profiler.stop()
+                        cprofiler.disable()
 
-                    now = datetime.now(timezone.utc)
+                    now = datetime.now(UTC)
                     log_path = os.path.dirname(payload_item.logfile)
                     profile_log_prefix = (
                         f"{log_path}/{payload_item['index']}_{payload_item['name']}_{now.strftime('%Y-%m-%d_%H-%M-%S')}"
                     )
 
-                    if customizer_settings.get("cprofiling", False):
+                    if customizer_settings.get("profiling", False):
                         import io
 
                         s = io.StringIO()
@@ -785,10 +752,6 @@ class PayloadList:
                             f.write(s.getvalue())
                         stats.dump_stats(filename=f"{profile_log_prefix}.cprof")
 
-                    if customizer_settings.get("profiling", False):
-                        with open(f"{profile_log_prefix}.html", "w") as f:
-                            f.write(profiler.output_html())
-
                 except ValidationError:
                     thread_logger.error("Validation error!")
                     success = False
@@ -818,7 +781,7 @@ class PayloadList:
                 # Update the status to "completed" in the DataFrame after processing finishes
                 self.update_payload_item(payload_item["index"], {"status": "completed"})
 
-            stop_time = datetime.now(timezone.utc)
+            stop_time = datetime.now(UTC)
             duration = stop_time - start_time
 
             # Format duration in hh:mm:ss

{pyxecm/customizer → pyxecm_customizer}/salesforce.py

@@ -17,7 +17,7 @@ from http import HTTPStatus
 
 import requests
 
-default_logger = logging.getLogger("pyxecm.customizer.salesforce")
+default_logger = logging.getLogger("pyxecm_customizer.salesforce")
 
 REQUEST_LOGIN_HEADERS = {
     "Content-Type": "application/x-www-form-urlencoded",

{pyxecm/customizer → pyxecm_customizer}/sap.py

@@ -32,7 +32,7 @@ __email__ = "mdiefenb@opentext.com"
 
 import logging
 
-default_logger = logging.getLogger("pyxecm.customizer.sap")
+default_logger = logging.getLogger("pyxecm_customizer.sap")
 
 try:
     import pyrfc
@@ -79,7 +79,11 @@ class SAP:
                 self.logger.addFilter(logfilter)
 
         self.logger.info("Initializing SAP object...")
-        self.logger.info("Using PyRFC version -> %s", pyrfc.__version__)
+
+        if _has_pyrfc:
+            self.logger.info("Using PyRFC version -> %s", pyrfc.__version__)
+        else:
+            self.logger.warning("PyRFC not installed. Cannot talk to SAP!")
 
         # Set up connection parameters
         self._connection_parameters = {

{pyxecm/customizer → pyxecm_customizer}/servicenow.py

@@ -24,11 +24,10 @@ from importlib.metadata import version
 from typing import Any
 
 import requests
+from pyxecm.helper import Data
 from requests.auth import HTTPBasicAuth
 from requests.exceptions import HTTPError, RequestException
 
-from pyxecm.helper import Data
-
 APP_NAME = "pyxecm"
 APP_VERSION = version("pyxecm")
 MODULE_NAME = APP_NAME + ".customizer.servicenow"
@@ -1120,8 +1119,7 @@ class ServiceNow:
 
                 # Read and write the attachment file in chunks:
                 with open(file_path, "wb") as attachment_file:
-                    for chunk in attachment_response.iter_content(chunk_size=8192):
-                        attachment_file.write(chunk)
+                    attachment_file.writelines(attachment_response.iter_content(chunk_size=8192))
 
                 # We build a list of filenames and IDs.
                 # The IDs we want to use as nicknames later on.

{pyxecm/customizer → pyxecm_customizer}/settings.py

@@ -265,6 +265,8 @@ class CustomizerSettingsK8S(BaseModel):
     sts_otcs_frontend_replicas: int = Field(None)
     sts_otcs_admin: str = Field(default="otcs-admin", description="Name of the OTCS-ADMIN statefulset")
     sts_otcs_admin_replicas: int = Field(default=None)
+    sts_otcs_da: str = Field(default="otcs-da", description="Name of the OTCS-DA statefulset")
+    sts_otcs_da_replicas: int = Field(default=None)
     ingress_otxecm: str = Field(default="otxecm-ingress", description="Name of the otxecm ingress")
 
     maintenance_service_name: str = Field(default="customizer")
@@ -348,10 +350,13 @@ class CustomizerSettingsAviator(BaseModel):
     oauth_client: str = Field(default="", description="OAuth Client ID for Content Aviator")
     oauth_secret: str = Field(default="", description="OAuth Client Secret for Content Aviator")
     chat_svc_url: HttpUrl = Field(
-        default="http://csai-chat-svc:3000", description="Chat Service URL for Content Aviator"
+        default=HttpUrl("http://csai-chat-svc:3000"), description="Chat Service URL for Content Aviator"
     )
     embed_svc_url: HttpUrl = Field(
-        default="http://csai-embed-svc:3000", description="Embed Service URL for Content Aviator"
+        default=HttpUrl("http://csai-embed-svc:3000"), description="Embed Service URL for Content Aviator"
+    )
+    studio_url: HttpUrl = Field(
+        default=HttpUrl("http://csai-aviator-studio"), description="Service URL for Aviator Studio"
     )
 
 
@@ -416,10 +421,6 @@ class Settings(BaseSettings):
     placeholder_values: dict = Field(default={})
 
     profiling: bool = Field(
-        default=False,
-        description="Profiling can only be enabled when using the CustomizerAPI. Switch to enable python profiling using pyinstrument. Result is a html file showing the execution of payload broken down into functions and their duration. The files are located in the logdir. Profiling is disabled by default.",
-    )
-    cprofiling: bool = Field(
         default=False,
         description="Profiling can only be enabled when using the CustomizerAPI. Switch to enable python profiling using cProfile. Result is a log file with the cProfile results, as well as a dump of the profiling session. The files are located in the logdir. The files are located in the logdir. Profilig is disabled by default.",
     )

{pyxecm/customizer → pyxecm_customizer}/successfactors.py

@@ -19,7 +19,7 @@ import urllib.parse
 import requests
 import xmltodict
 
-default_logger = logging.getLogger("pyxecm.customizer.sucessfactors")
+default_logger = logging.getLogger("pyxecm_customizer.sucessfactors")
 
 request_login_headers = {
     "Content-Type": "application/x-www-form-urlencoded",  # "application/json",
@@ -63,11 +63,11 @@ class SuccessFactors:
                 The SuccessFactors Client ID.
             client_secret (str):
                 The SuccessFactors Client Secret.
-            username (str):
+            username (str, optional):
                 The user name in SuccessFactors.
-            password (str):
+            password (str, optional):
                 The password of the SuccessFactors user.
-            company_id (str):
+            company_id (str, optional):
                 The SuccessFactors company ID.
             authorization_url (str, optional):
                 URL for SuccessFactors login.
@@ -254,11 +254,16 @@ class SuccessFactors:
         """Check existence of key / value pair in the response properties of an SuccessFactors API call.
 
         Args:
-            response (dict): REST response from an SuccessFactors API call
-            key (str): property name (key)
-            value (str): value to find in the item with the matching key
+            response (dict):
+                REST response from an SuccessFactors API call
+            key (str):
+                The property name (key).
+            value (str):
+                The value to find in the item with the matching key.
+
         Returns:
-            bool: True if the value was found, False otherwise
+            bool:
+                True if the value was found, False otherwise
 
         """
 
@@ -290,13 +295,17 @@ class SuccessFactors:
         """Get value of a result property with a given key of an SuccessFactors API call.
 
         Args:
-            response (dict): REST response from an SuccessFactors REST Call
-            key (str): property name (key)
-            index (int, optional): Index to use (1st element has index 0).
-                                   Defaults to 0.
+            response (dict):
+                REST response from an SuccessFactors REST Call
+            key (str):
+                The property name (key).
+            index (int, optional):
+                Index to use (1st element has index 0).
+                Defaults to 0.
 
         Returns:
-            str: value for the key, None otherwise
+            str | None:
+                The value for the key, None if not found.
 
         """
 
@@ -342,7 +351,9 @@ class SuccessFactors:
         Args:
             None
         Returns:
-            str: Assertion. Also stores access token in self._assertion. None in case of error
+            str:
+                The SAML assertion. Also stores access token in self._assertion.
+                Returns None in case of an error.
 
         """
 
@@ -460,7 +471,7 @@ class SuccessFactors:
         """Get information for a country / countries.
 
         Args:
-            code (str):
+            code (str, optional):
                 3 character code for contry selection, like "USA"
 
         Returns:
@@ -545,13 +556,14 @@ class SuccessFactors:
                 The field name of the filter.
             field_value (str):
                 The filter value to compare the field with.
-            field_operation (str):
+            field_operation (str, optional):
                 The operation of the filter. Like "in".
-            max_results (int):
+            max_results (int, optional):
                 The maximum number of results to return. Default is 1.
 
         Returns:
-            dict | None: User Account details
+            dict | None:
+                User Account details
 
             Example return data in "d" dictionary:
 
@@ -817,14 +829,14 @@ class SuccessFactors:
             entity (str, optional):
                 Entity type to query. Examples are "PerPerson" (default),
                 "PerPersonal", "PerEmail", "PersonKey", ...
-            field_name (str):
+            field_name (str, optional):
                 Field to search in. E.g. personIdExternal, firstName, lastName,
                 fullName, email, dateOfBirth, gender, nationality, maritalStatus,
                 employeeId.
-            field_value (str):
+            field_value (str, optional):
                 Value to match in the Field
-            field_operation (str):
-                The operation to apply for the filter.
+            field_operation (str, optional):
+                The operation to apply for the filter. Default is 'eq' (equal).
             max_results (int):
                 The maximum number of results to return. Default is 1.
 
@@ -972,7 +984,7 @@ class SuccessFactors:
         The metadata response supports only application/xml type.
 
         Args:
-            entities (list):
+            entities (list | None, optional):
                 A list of entities to deliver metadata for.
 
         Returns:
@@ -1055,8 +1067,8 @@ class SuccessFactors:
         self,
         user_id: str,  # this is NOT the username but really an ID like 106020
         email_address: str,
-        email_type: int = 8448,  # 8448
-    ) -> dict:
+        email_type: int = 8448,
+    ) -> dict | None:
         """Update user email.
 
         See: https://help.sap.com/docs/SAP_SUCCESSFACTORS_PLATFORM/d599f15995d348a1b45ba5603e2aba9b/7b3daeb3d77d491bb401345eede34bb5.html?locale=en-US
@@ -1066,11 +1078,11 @@ class SuccessFactors:
                 The ID of the user (e.g. 106020).
             email_address (str):
                 The new email address of user.
-            email_type (int):
-                Type of the email. 8448 = Business
+            email_type (int, optional):
+                Type of the email. 8448 = Business.
 
         Returns:
-            dict:
+            dict | None:
                 Request response or None if an error occured.
 
         """

{pyxecm/customizer → pyxecm_customizer}/translate.py

@@ -10,7 +10,7 @@ import logging
 
 import requests
 
-default_logger = logging.getLogger("pyxecm.customizer.translate")
+default_logger = logging.getLogger("pyxecm_customizer.translate")
 
 REQUEST_TIMEOUT = 60
 

{pyxecm/maintenance_page → pyxecm_maintenance_page}/__main__.py

@@ -1,6 +1,6 @@
 """Wrapper to start the Maintenance Page."""
 
-from pyxecm.maintenance_page import run_maintenance_page
+from pyxecm_maintenance_page import run_maintenance_page
 
 if __name__ == "__main__":
     run_maintenance_page()

{pyxecm/maintenance_page → pyxecm_maintenance_page}/app.py

@@ -1,17 +1,19 @@
 """Maintenance Page that can be enabled by the customizer."""
 
+import logging
 import os
 import threading
-from datetime import datetime
+from datetime import UTC, datetime
 
 import uvicorn
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import FileResponse
 from fastapi.templating import Jinja2Templates
-from pytz import UTC
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
-from pyxecm.maintenance_page.settings import settings
+from .settings import settings
+
+logger = logging.getLogger("pyxecm_customizer.maintenance_page")
 
 app = FastAPI(openapi_url=None)
 
@@ -39,7 +41,7 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Jinja2
                 "maint_text": settings.text,
                 "maint_footer": settings.footer,
                 "status_url": settings.status_url,
-                "copyright_year": datetime.now(tz=UTC).year,
+                "copyright_year": datetime.now(UTC).year,
             },
             status_code=513,
         )
@@ -49,7 +51,11 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Jinja2
 
 def run_maintenance_page() -> None:
     """Start the FASTAPI Webserver in a dedicated thread."""
-    maint_thread = threading.Thread(
-        target=uvicorn.run, name="MaintenancePage", kwargs={"app": app, "host": settings.host, "port": settings.port}
-    )
-    maint_thread.start()
+
+    def start_server() -> None:
+        try:
+            uvicorn.run(app, host=settings.host, port=settings.port)
+        except Exception:
+            logger.error("Could not start the maintenance page.")
+
+    threading.Thread(target=start_server, name="MaintenancePage").start()

pyxecm/customizer/api/app.py

@@ -1,157 +0,0 @@
-"""API Implemenation for the Customizer to start and control the payload processing."""
-
-__author__ = "Dr. Marc Diefenbruch"
-__copyright__ = "Copyright (C) 2024-2025, OpenText"
-__credits__ = ["Kai-Philip Gatzweiler"]
-__maintainer__ = "Dr. Marc Diefenbruch"
-__email__ = "mdiefenb@opentext.com"
-
-import logging
-import os
-import sys
-from collections.abc import AsyncGenerator
-from contextlib import asynccontextmanager
-from datetime import datetime, timezone
-from importlib.metadata import version
-
-import uvicorn
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-from prometheus_fastapi_instrumentator import Instrumentator
-
-from pyxecm.customizer.api.auth.router import router as auth_router
-from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
-from pyxecm.customizer.api.common.metrics import payload_logs_by_payload, payload_logs_total
-from pyxecm.customizer.api.common.router import router as common_router
-from pyxecm.customizer.api.settings import api_settings
-from pyxecm.customizer.api.terminal.router import router as terminal_router
-from pyxecm.customizer.api.v1_csai.router import router as v1_csai_router
-from pyxecm.customizer.api.v1_maintenance.router import router as v1_maintenance_router
-from pyxecm.customizer.api.v1_otcs.router import router as v1_otcs_router
-from pyxecm.customizer.api.v1_payload.functions import import_payload
-from pyxecm.customizer.api.v1_payload.router import router as v1_payload_router
-from pyxecm.maintenance_page import run_maintenance_page
-
-# Check if Temp dir exists
-if not os.path.exists(api_settings.temp_dir):
-    os.makedirs(api_settings.temp_dir)
-
-# Check if Logfile and folder exists and is unique
-if os.path.isfile(os.path.join(api_settings.logfolder, api_settings.logfile)):
-    customizer_start_time = datetime.now(timezone.utc).strftime(
-        "%Y-%m-%d_%H-%M",
-    )
-    api_settings.logfile = f"customizer_{customizer_start_time}.log"
-elif not os.path.exists(api_settings.logfolder):
-    os.makedirs(api_settings.logfolder)
-
-handlers = [
-    logging.FileHandler(os.path.join(api_settings.logfolder, api_settings.logfile)),
-    logging.StreamHandler(sys.stdout),
-]
-
-logging.basicConfig(
-    format="%(asctime)s %(levelname)s [%(name)s] [%(threadName)s] %(message)s",
-    datefmt="%d-%b-%Y %H:%M:%S",
-    level=api_settings.loglevel,
-    handlers=handlers,
-)
-
-
-@asynccontextmanager
-async def lifespan(
-    app: FastAPI,  # noqa: ARG001
-) -> AsyncGenerator:
-    """Lifespan Method for FASTAPI to handle the startup and shutdown process.
-
-    Args:
-        app (FastAPI):
-            The application.
-
-    """
-
-    logger.debug("Settings -> %s", api_settings)
-
-    if api_settings.import_payload:
-        logger.info("Importing filesystem payloads...")
-
-        # Base Payload
-        import_payload(payload=api_settings.payload)
-
-        # External Payload
-        import_payload(payload_dir=api_settings.payload_dir, dependencies=True)
-
-        # Optional Payload
-        import_payload(payload_dir=api_settings.payload_dir_optional)
-
-    logger.info("Starting maintenance_page thread...")
-    run_maintenance_page()
-
-    logger.info("Starting processing thread...")
-    PAYLOAD_LIST.run_payload_processing(concurrent=api_settings.concurrent_payloads)
-
-    yield
-    logger.info("Shutdown")
-    PAYLOAD_LIST.stop_payload_processing()
-
-
-app = FastAPI(
-    docs_url="/api",
-    title=api_settings.title,
-    description=api_settings.description,
-    openapi_url=api_settings.openapi_url,
-    root_path=api_settings.root_path,
-    lifespan=lifespan,
-    version=version("pyxecm"),
-    openapi_tags=[
-        {
-            "name": "auth",
-            "description": "Authentication Endpoint - Users are authenticated against Opentext Directory Services",
-        },
-        {
-            "name": "payload",
-            "description": "Get status and manipulate payload objects ",
-        },
-        {
-            "name": "maintenance",
-            "description": "Enable, disable or alter the maintenance mode.",
-        },
-    ],
-)
-
-## Add all Routers
-app.include_router(router=common_router)
-app.include_router(router=auth_router)
-app.include_router(router=v1_maintenance_router)
-app.include_router(router=v1_otcs_router)
-app.include_router(router=v1_payload_router)
-if api_settings.ws_terminal:
-    app.include_router(router=terminal_router)
-if api_settings.csai:
-    app.include_router(router=v1_csai_router)
-
-logger = logging.getLogger("CustomizerAPI")
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=api_settings.trusted_origins,
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-if api_settings.metrics:
-    # Add Prometheus Instrumentator for /metrics
-    instrumentator = Instrumentator().instrument(app).expose(app)
-    instrumentator.add(payload_logs_by_payload(PAYLOAD_LIST))
-    instrumentator.add(payload_logs_total(PAYLOAD_LIST))
-
-
-def run_api() -> None:
-    """Start the FASTAPI Webserver."""
-
-    uvicorn.run(
-        "pyxecm.customizer.api:app",
-        host=api_settings.bind_address,
-        port=api_settings.bind_port,
-        workers=api_settings.workers,
-    )