pyxecm 2.0.3__py3-none-any.whl → 3.0.0__py3-none-any.whl

{pyxecm/customizer/api/common → pyxecm_customizer}/payload_list.py

@@ -15,29 +15,19 @@ import pprint
 import threading
 import time
 import traceback
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
+import pandas as pd
 from pydantic import ValidationError
-
-from pyxecm.customizer.api.settings import api_settings
+from pyxecm.helper.otel_config import trace, tracer
 
 # OpenText specific modules:
-from pyxecm.customizer.customizer import Customizer
-from pyxecm.customizer.exceptions import StopOnError
-from pyxecm.customizer.log import LogCountFilter, VictoriaLogsHandler
-from pyxecm.customizer.payload import load_payload
-
-default_logger = logging.getLogger("pyxecm.customizer.payload_list")
-
-try:
-    import pandas as pd
+from pyxecm_customizer.customizer import Customizer
+from pyxecm_customizer.exceptions import StopOnError
+from pyxecm_customizer.log import LogCountFilter
+from pyxecm_customizer.payload import load_payload
 
-    pandas_installed = True
-except ModuleNotFoundError:
-    default_logger.warning(
-        "Module pandas is not installed. Customizer will not support bulk workspace creation.",
-    )
-    pandas_installed = False
+default_logger = logging.getLogger("pyxecm_customizer.payload_list")
 
 
 class PayloadList:
@@ -93,7 +83,7 @@ class PayloadList:
 
         def calculate_duration(row: pd.Series) -> str:
             if row["status"] == "running":
-                now = datetime.now(timezone.utc)
+                now = datetime.now(UTC)
                 start_time = pd.to_datetime(row["start_time"])
 
                 duration = now - start_time
@@ -609,7 +599,7 @@ class PayloadList:
 
         # Log each runnable item
         for _, row in runnable_df.iterrows():
-            self.logger.info(
+            self.logger.debug(
                 "Added payload file -> '%s' with index -> %s to runnable queue.",
                 row["name"] if row["name"] else row["filename"],
                 row["index"],
@@ -619,18 +609,45 @@ class PayloadList:
 
     # end method definition
 
-    def process_payload_list(self) -> None:
+    def pick_running(self) -> int:
+        """Pick all PayloadItems with status "running".
+
+        Returns:
+            pd.DataFrame:
+                A list of running payload items.
+
+        """
+
+        if self.payload_items.empty:
+            return 0
+
+        all_status = self.payload_items["status"].value_counts().to_dict()
+
+        return all_status.get("running", 0)
+
+    # end method definition
+
+    def process_payload_list(self, concurrent: int | None = None) -> None:
         """Process runnable payloads.
 
+        Args:
+            concurrent (int | None, optional):
+                The maximum number of concurrent payloads to run at any given time.
+
         Continuously checks for runnable payload items and starts their
         "process_payload" method in separate threads.
         Runs as a daemon until the customizer ends.
+
         """
 
+        @tracer.start_as_current_span("run_and_complete_payload")
         def run_and_complete_payload(payload_item: pd.Series) -> None:
             """Run the payload's process_payload method and marks the status as completed afterward."""
 
-            start_time = datetime.now(timezone.utc)
+            t = trace.get_current_span()
+            t.set_attributes({"payload_id": payload_item["index"], "payload_name": payload_item["name"]})
+
+            start_time = datetime.now(UTC)
             self.update_payload_item(payload_item["index"], {"start_time": start_time})
 
             # Create a logger with thread_id:
@@ -656,24 +673,6 @@ class PayloadList:
             handler.setFormatter(fmt=formatter)
             thread_logger.addHandler(hdlr=handler)
 
-            # If hostname is set, configure log handler so forward logs
-            if api_settings.victorialogs_host:
-                handler_kwargs = {
-                    "host": api_settings.victorialogs_host,
-                    "port": api_settings.victorialogs_port,
-                    "app": "Customizer",
-                    "payload_item": payload_item["index"],
-                    "payload_file": payload_item["filename"],
-                }
-
-                # Read namespace if available and add as kwarg to loghandler
-                file_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
-                if os.path.isfile(file_path):
-                    with open(file_path) as file:
-                        handler_kwargs["namespace"] = file.read()
-
-                thread_logger.addHandler(VictoriaLogsHandler(**handler_kwargs))
-
             if len(thread_logger.filters) == 0:
                 thread_logger.debug("Adding log count filter to logger")
                 thread_logger.addFilter(
@@ -726,12 +725,6 @@ class PayloadList:
                     )
 
                     if customizer_settings.get("profiling", False):
-                        from pyinstrument import Profiler
-
-                        profiler = Profiler()
-                        profiler.start()
-
-                    if customizer_settings.get("cprofiling", False):
                         import cProfile
                         import pstats
 
@@ -740,19 +733,16 @@ class PayloadList:
 
                     success = local.customizer_thread_object.customization_run()
 
-                    if customizer_settings.get("cprofiling", False):
-                        cprofiler.disable()
-
                     if customizer_settings.get("profiling", False):
-                        profiler.stop()
+                        cprofiler.disable()
 
-                    now = datetime.now(timezone.utc)
+                    now = datetime.now(UTC)
                     log_path = os.path.dirname(payload_item.logfile)
                     profile_log_prefix = (
                         f"{log_path}/{payload_item['index']}_{payload_item['name']}_{now.strftime('%Y-%m-%d_%H-%M-%S')}"
                     )
 
-                    if customizer_settings.get("cprofiling", False):
+                    if customizer_settings.get("profiling", False):
                         import io
 
                         s = io.StringIO()
@@ -762,10 +752,6 @@ class PayloadList:
                             f.write(s.getvalue())
                         stats.dump_stats(filename=f"{profile_log_prefix}.cprof")
 
-                    if customizer_settings.get("profiling", False):
-                        with open(f"{profile_log_prefix}.html", "w") as f:
-                            f.write(profiler.output_html())
-
                 except ValidationError:
                     thread_logger.error("Validation error!")
                     success = False
@@ -795,7 +781,7 @@ class PayloadList:
                 # Update the status to "completed" in the DataFrame after processing finishes
                 self.update_payload_item(payload_item["index"], {"status": "completed"})
 
-            stop_time = datetime.now(timezone.utc)
+            stop_time = datetime.now(UTC)
             duration = stop_time - start_time
 
             # Format duration in hh:mm:ss
@@ -817,6 +803,18 @@ class PayloadList:
             # Start a thread for each runnable item (item is a pd.Series)
             if runnable_items is not None:
                 for _, item in runnable_items.iterrows():
+                    if concurrent and self.pick_running() >= concurrent:
+                        self.logger.debug(
+                            "Reached concurrency limit of %s payloads. Waiting for one to finish.",
+                        )
+                        break
+
+                    self.logger.info(
+                        "Added payload file -> '%s' with index -> %s to runnable queue.",
+                        item["name"] if item["name"] else item["filename"],
+                        item["index"],
+                    )
+
                     # Update the status to "running" in the data frame to prevent re-processing
                     self.payload_items.loc[
                         self.payload_items["name"] == item["name"],
@@ -837,13 +835,14 @@ class PayloadList:
 
     # end method definition
 
-    def run_payload_processing(self) -> None:
+    def run_payload_processing(self, concurrent: int | None = None) -> None:
         """Start the `process_payload_list` method in a daemon thread."""
 
         scheduler_thread = threading.Thread(
             target=self.process_payload_list,
             daemon=True,
             name="Scheduler",
+            kwargs={"concurrent": concurrent},
         )
 
         self.logger.info(
@@ -853,13 +852,6 @@ class PayloadList:
         self._stopped = False
         scheduler_thread.start()
 
-        self.logger.info(
-            "Waiting for thread -> '%s' to complete...",
-            str(scheduler_thread.name),
-        )
-        scheduler_thread.join()
-        self.logger.info("Thread -> '%s' has completed.", str(scheduler_thread.name))
-
     # end method definition
 
     def stop_payload_processing(self) -> None:

{pyxecm/customizer → pyxecm_customizer}/salesforce.py

@@ -17,7 +17,7 @@ from http import HTTPStatus
 
 import requests
 
-default_logger = logging.getLogger("pyxecm.customizer.salesforce")
+default_logger = logging.getLogger("pyxecm_customizer.salesforce")
 
 REQUEST_LOGIN_HEADERS = {
     "Content-Type": "application/x-www-form-urlencoded",

{pyxecm/customizer → pyxecm_customizer}/sap.py

@@ -32,7 +32,7 @@ __email__ = "mdiefenb@opentext.com"
 
 import logging
 
-default_logger = logging.getLogger("pyxecm.customizer.sap")
+default_logger = logging.getLogger("pyxecm_customizer.sap")
 
 try:
     import pyrfc
@@ -79,7 +79,11 @@ class SAP:
                 self.logger.addFilter(logfilter)
 
         self.logger.info("Initializing SAP object...")
-        self.logger.info("Using PyRFC version -> %s", pyrfc.__version__)
+
+        if _has_pyrfc:
+            self.logger.info("Using PyRFC version -> %s", pyrfc.__version__)
+        else:
+            self.logger.warning("PyRFC not installed. Cannot talk to SAP!")
 
         # Set up connection parameters
         self._connection_parameters = {

{pyxecm/customizer → pyxecm_customizer}/servicenow.py

@@ -24,11 +24,10 @@ from importlib.metadata import version
 from typing import Any
 
 import requests
+from pyxecm.helper import Data
 from requests.auth import HTTPBasicAuth
 from requests.exceptions import HTTPError, RequestException
 
-from pyxecm.helper import Data
-
 APP_NAME = "pyxecm"
 APP_VERSION = version("pyxecm")
 MODULE_NAME = APP_NAME + ".customizer.servicenow"
@@ -1120,8 +1119,7 @@ class ServiceNow:
 
                 # Read and write the attachment file in chunks:
                 with open(file_path, "wb") as attachment_file:
-                    for chunk in attachment_response.iter_content(chunk_size=8192):
-                        attachment_file.write(chunk)
+                    attachment_file.writelines(attachment_response.iter_content(chunk_size=8192))
 
                 # We build a list of filenames and IDs.
                 # The IDs we want to use as nicknames later on.

{pyxecm/customizer → pyxecm_customizer}/settings.py

@@ -265,6 +265,8 @@ class CustomizerSettingsK8S(BaseModel):
     sts_otcs_frontend_replicas: int = Field(None)
     sts_otcs_admin: str = Field(default="otcs-admin", description="Name of the OTCS-ADMIN statefulset")
     sts_otcs_admin_replicas: int = Field(default=None)
+    sts_otcs_da: str = Field(default="otcs-da", description="Name of the OTCS-DA statefulset")
+    sts_otcs_da_replicas: int = Field(default=None)
     ingress_otxecm: str = Field(default="otxecm-ingress", description="Name of the otxecm ingress")
 
     maintenance_service_name: str = Field(default="customizer")
@@ -348,10 +350,13 @@ class CustomizerSettingsAviator(BaseModel):
     oauth_client: str = Field(default="", description="OAuth Client ID for Content Aviator")
     oauth_secret: str = Field(default="", description="OAuth Client Secret for Content Aviator")
     chat_svc_url: HttpUrl = Field(
-        default="http://csai-chat-svc:3000", description="Chat Service URL for Content Aviator"
+        default=HttpUrl("http://csai-chat-svc:3000"), description="Chat Service URL for Content Aviator"
     )
     embed_svc_url: HttpUrl = Field(
-        default="http://csai-embed-svc:3000", description="Embed Service URL for Content Aviator"
+        default=HttpUrl("http://csai-embed-svc:3000"), description="Embed Service URL for Content Aviator"
+    )
+    studio_url: HttpUrl = Field(
+        default=HttpUrl("http://csai-aviator-studio"), description="Service URL for Aviator Studio"
     )
 
 
@@ -416,10 +421,6 @@ class Settings(BaseSettings):
     placeholder_values: dict = Field(default={})
 
     profiling: bool = Field(
-        default=False,
-        description="Profiling can only be enabled when using the CustomizerAPI. Switch to enable python profiling using pyinstrument. Result is a html file showing the execution of payload broken down into functions and their duration. The files are located in the logdir. Profiling is disabled by default.",
-    )
-    cprofiling: bool = Field(
         default=False,
         description="Profiling can only be enabled when using the CustomizerAPI. Switch to enable python profiling using cProfile. Result is a log file with the cProfile results, as well as a dump of the profiling session. The files are located in the logdir. The files are located in the logdir. Profilig is disabled by default.",
     )

{pyxecm/customizer → pyxecm_customizer}/successfactors.py

@@ -19,7 +19,7 @@ import urllib.parse
 import requests
 import xmltodict
 
-default_logger = logging.getLogger("pyxecm.customizer.sucessfactors")
+default_logger = logging.getLogger("pyxecm_customizer.sucessfactors")
 
 request_login_headers = {
     "Content-Type": "application/x-www-form-urlencoded",  # "application/json",
@@ -63,11 +63,11 @@ class SuccessFactors:
                 The SuccessFactors Client ID.
             client_secret (str):
                 The SuccessFactors Client Secret.
-            username (str):
+            username (str, optional):
                 The user name in SuccessFactors.
-            password (str):
+            password (str, optional):
                 The password of the SuccessFactors user.
-            company_id (str):
+            company_id (str, optional):
                 The SuccessFactors company ID.
             authorization_url (str, optional):
                 URL for SuccessFactors login.
@@ -254,11 +254,16 @@ class SuccessFactors:
         """Check existence of key / value pair in the response properties of an SuccessFactors API call.
 
         Args:
-            response (dict): REST response from an SuccessFactors API call
-            key (str): property name (key)
-            value (str): value to find in the item with the matching key
+            response (dict):
+                REST response from an SuccessFactors API call
+            key (str):
+                The property name (key).
+            value (str):
+                The value to find in the item with the matching key.
+
         Returns:
-            bool: True if the value was found, False otherwise
+            bool:
+                True if the value was found, False otherwise
 
         """
 
@@ -290,13 +295,17 @@ class SuccessFactors:
         """Get value of a result property with a given key of an SuccessFactors API call.
 
         Args:
-            response (dict): REST response from an SuccessFactors REST Call
-            key (str): property name (key)
-            index (int, optional): Index to use (1st element has index 0).
-                                   Defaults to 0.
+            response (dict):
+                REST response from an SuccessFactors REST Call
+            key (str):
+                The property name (key).
+            index (int, optional):
+                Index to use (1st element has index 0).
+                Defaults to 0.
 
         Returns:
-            str: value for the key, None otherwise
+            str | None:
+                The value for the key, None if not found.
 
         """
 
@@ -342,7 +351,9 @@ class SuccessFactors:
         Args:
             None
         Returns:
-            str: Assertion. Also stores access token in self._assertion. None in case of error
+            str:
+                The SAML assertion. Also stores access token in self._assertion.
+                Returns None in case of an error.
 
         """
 
@@ -460,7 +471,7 @@ class SuccessFactors:
         """Get information for a country / countries.
 
         Args:
-            code (str):
+            code (str, optional):
                 3 character code for contry selection, like "USA"
 
         Returns:
@@ -545,13 +556,14 @@ class SuccessFactors:
                 The field name of the filter.
             field_value (str):
                 The filter value to compare the field with.
-            field_operation (str):
+            field_operation (str, optional):
                 The operation of the filter. Like "in".
-            max_results (int):
+            max_results (int, optional):
                 The maximum number of results to return. Default is 1.
 
         Returns:
-            dict | None: User Account details
+            dict | None:
+                User Account details
 
             Example return data in "d" dictionary:
 
@@ -817,14 +829,14 @@ class SuccessFactors:
             entity (str, optional):
                 Entity type to query. Examples are "PerPerson" (default),
                 "PerPersonal", "PerEmail", "PersonKey", ...
-            field_name (str):
+            field_name (str, optional):
                 Field to search in. E.g. personIdExternal, firstName, lastName,
                 fullName, email, dateOfBirth, gender, nationality, maritalStatus,
                 employeeId.
-            field_value (str):
+            field_value (str, optional):
                 Value to match in the Field
-            field_operation (str):
-                The operation to apply for the filter.
+            field_operation (str, optional):
+                The operation to apply for the filter. Default is 'eq' (equal).
             max_results (int):
                 The maximum number of results to return. Default is 1.
 
@@ -972,7 +984,7 @@ class SuccessFactors:
         The metadata response supports only application/xml type.
 
         Args:
-            entities (list):
+            entities (list | None, optional):
                 A list of entities to deliver metadata for.
 
         Returns:
@@ -1055,8 +1067,8 @@ class SuccessFactors:
         self,
         user_id: str,  # this is NOT the username but really an ID like 106020
         email_address: str,
-        email_type: int = 8448,  # 8448
-    ) -> dict:
+        email_type: int = 8448,
+    ) -> dict | None:
         """Update user email.
 
         See: https://help.sap.com/docs/SAP_SUCCESSFACTORS_PLATFORM/d599f15995d348a1b45ba5603e2aba9b/7b3daeb3d77d491bb401345eede34bb5.html?locale=en-US
@@ -1066,11 +1078,11 @@ class SuccessFactors:
                 The ID of the user (e.g. 106020).
             email_address (str):
                 The new email address of user.
-            email_type (int):
-                Type of the email. 8448 = Business
+            email_type (int, optional):
+                Type of the email. 8448 = Business.
 
         Returns:
-            dict:
+            dict | None:
                 Request response or None if an error occured.
 
         """

{pyxecm/customizer → pyxecm_customizer}/translate.py

@@ -10,7 +10,7 @@ import logging
 
 import requests
 
-default_logger = logging.getLogger("pyxecm.customizer.translate")
+default_logger = logging.getLogger("pyxecm_customizer.translate")
 
 REQUEST_TIMEOUT = 60
 
@@ -149,16 +149,20 @@ class Translator:
         request_header = self._headers
         request_url = self.config()["translateUrlV3"]
 
-        response = requests.post(
-            url=request_url,
-            headers=request_header,
-            json=data,
-            timeout=REQUEST_TIMEOUT,
-        )
+        try:
+            response = requests.post(
+                url=request_url,
+                headers=request_header,
+                json=data,
+                timeout=REQUEST_TIMEOUT,
+            )
 
-        if response.status_code != 200:
-            self.logger.error("Failed to translate text -> %s", response.content)
-            return None
+            if response.status_code != 200:
+                self.logger.error("Failed to translate text -> %s", response.content)
+                return None
+
+        except Exception as error:
+            self.logger.error("Failed translation request; error -> %s", str(error))
 
         translated_text = response.json()["data"]["translations"][0]["translatedText"]
 

{pyxecm/maintenance_page → pyxecm_maintenance_page}/__main__.py

@@ -1,6 +1,6 @@
 """Wrapper to start the Maintenance Page."""
 
-from pyxecm.maintenance_page import run_maintenance_page
+from pyxecm_maintenance_page import run_maintenance_page
 
 if __name__ == "__main__":
     run_maintenance_page()

{pyxecm/maintenance_page → pyxecm_maintenance_page}/app.py

@@ -1,16 +1,19 @@
 """Maintenance Page that can be enabled by the customizer."""
 
+import logging
 import os
-from datetime import datetime
+import threading
+from datetime import UTC, datetime
 
 import uvicorn
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import FileResponse
 from fastapi.templating import Jinja2Templates
-from pytz import UTC
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
-from pyxecm.maintenance_page.settings import settings
+from .settings import settings
+
+logger = logging.getLogger("pyxecm_customizer.maintenance_page")
 
 app = FastAPI(openapi_url=None)
 
@@ -38,7 +41,7 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Jinja2
                 "maint_text": settings.text,
                 "maint_footer": settings.footer,
                 "status_url": settings.status_url,
-                "copyright_year": datetime.now(tz=UTC).year,
+                "copyright_year": datetime.now(UTC).year,
             },
             status_code=513,
         )
@@ -47,5 +50,12 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Jinja2
 
 
 def run_maintenance_page() -> None:
-    """Start the FASTAPI Webserver."""
-    uvicorn.run(app, host=settings.host, port=settings.port)
+    """Start the FASTAPI Webserver in a dedicated thread."""
+
+    def start_server() -> None:
+        try:
+            uvicorn.run(app, host=settings.host, port=settings.port)
+        except Exception:
+            logger.error("Could not start the maintenance page.")
+
+    threading.Thread(target=start_server, name="MaintenancePage").start()