pyxecm 2.0.3__py3-none-any.whl → 2.0.4__py3-none-any.whl

pyxecm/coreshare.py

@@ -27,6 +27,7 @@ import time
 import urllib.parse
 from http import HTTPStatus
 from importlib.metadata import version
+from urllib.parse import parse_qs, urlparse
 
 import requests
 
@@ -522,7 +523,7 @@ class CoreShare:
                 Can be used to provide a more specific error message
                 in case an error occurs.
             show_error (bool, optional):
-                True: write an error to the log file
+                True: write an error to the log file (this is the default)
                 False: write a warning to the log file
 
         Returns:
@@ -933,6 +934,71 @@ class CoreShare:
 
     # end method definition
 
+    def get_groups_iterator(
+        self,
+        count: int | None = None,
+    ) -> iter:
+        """Get an iterator object that can be used to traverse all Core Share groups.
+
+        Returning a generator avoids loading a large number of items into memory at once. Instead you
+        can iterate over the potential large list of Core Share groups.
+
+        Example usage:
+            groups = core_share_object.get_groups_iterator(page_size=10)
+            for group in groups:
+                logger.info("Traversing Core Share group -> %s", group["name"])
+
+        Args:
+            count (int | None, optional):
+                The chunk size for the number of groups returned by one
+                REST API call. If None, then a default of 250 is used.
+
+        Returns:
+            iter:
+                A generator yielding one OTDS group per iteration.
+                If the REST API fails, returns no value.
+
+        """
+
+        offset = 0
+
+        while True:
+            response = self.get_groups(
+                offset=offset,
+                count=count,
+            )
+            if not response or not response.get("results", []):
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+
+            # Yield users one at a time:
+            yield from response["results"]
+
+            # See if we have an additional result page.
+            # If not terminate the iterator and return
+            # no value.
+
+            next_page_url = response["_links"].get("next")
+            if not next_page_url:
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+            next_page_url = next_page_url.get("href")
+
+            # Extract the query string from the URL
+            query = urlparse(next_page_url).query
+
+            # Parse the query parameters into a dictionary
+            params = parse_qs(query)
+
+            # Get the 'offset' value as an integer (it's a list by default)
+            offset = int(params.get("offset", [0])[0])
+
+    # end method definition
+
     def add_group(
         self,
         group_name: str,
@@ -1276,7 +1342,7 @@ class CoreShare:
             dict | None:
                 Dictionary with the Core Share group data or None if the request fails.
 
-            Example result:
+            Example Response:
             {
                 'results': [
                     {
@@ -1343,15 +1409,15 @@ class CoreShare:
 
     # end method definition
 
-    def get_users(self) -> dict | None:
+    def get_users(self) -> list | None:
         """Get Core Share users.
 
         Args:
             None
 
         Returns:
-            dict | None:
-                Dictionary with the Core Share user data or None if the request fails.
+            list | None:
+                List with the Core Share user data or None if the request fails.
 
             Example response (it is a list!):
             [

pyxecm/customizer/api/app.py

@@ -13,7 +13,6 @@ from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from importlib.metadata import version
-from threading import Thread
 
 import uvicorn
 from fastapi import FastAPI
@@ -85,17 +84,11 @@ async def lifespan(
         # Optional Payload
         import_payload(payload_dir=api_settings.payload_dir_optional)
 
-    if api_settings.maintenance_mode:
-        logger.info("Starting maintenance_page thread...")
-        maint_thread = Thread(target=run_maintenance_page, name="maintenance_page")
-        maint_thread.start()
+    logger.info("Starting maintenance_page thread...")
+    run_maintenance_page()
 
     logger.info("Starting processing thread...")
-    thread = Thread(
-        target=PAYLOAD_LIST.run_payload_processing,
-        name="customization_run_api",
-    )
-    thread.start()
+    PAYLOAD_LIST.run_payload_processing(concurrent=api_settings.concurrent_payloads)
 
     yield
     logger.info("Shutdown")
@@ -104,7 +97,8 @@ async def lifespan(
 
 app = FastAPI(
     docs_url="/api",
-    title="Customizer API",
+    title=api_settings.title,
+    description=api_settings.description,
     openapi_url=api_settings.openapi_url,
     root_path=api_settings.root_path,
     lifespan=lifespan,

pyxecm/customizer/api/common/payload_list.py

@@ -609,7 +609,7 @@ class PayloadList:
 
         # Log each runnable item
         for _, row in runnable_df.iterrows():
-            self.logger.info(
+            self.logger.debug(
                 "Added payload file -> '%s' with index -> %s to runnable queue.",
                 row["name"] if row["name"] else row["filename"],
                 row["index"],
@@ -619,12 +619,35 @@ class PayloadList:
 
     # end method definition
 
-    def process_payload_list(self) -> None:
+    def pick_running(self) -> int:
+        """Pick all PayloadItems with status "running".
+
+        Returns:
+            pd.DataFrame:
+                A list of running payload items.
+
+        """
+
+        if self.payload_items.empty:
+            return 0
+
+        all_status = self.payload_items["status"].value_counts().to_dict()
+
+        return all_status.get("running", 0)
+
+    # end method definition
+
+    def process_payload_list(self, concurrent: int | None = None) -> None:
         """Process runnable payloads.
 
+        Args:
+            concurrent (int | None, optional):
+                The maximum number of concurrent payloads to run at any given time.
+
         Continuously checks for runnable payload items and starts their
         "process_payload" method in separate threads.
         Runs as a daemon until the customizer ends.
+
         """
 
         def run_and_complete_payload(payload_item: pd.Series) -> None:
@@ -817,6 +840,18 @@ class PayloadList:
             # Start a thread for each runnable item (item is a pd.Series)
             if runnable_items is not None:
                 for _, item in runnable_items.iterrows():
+                    if concurrent and self.pick_running() >= concurrent:
+                        self.logger.debug(
+                            "Reached concurrency limit of %s payloads. Waiting for one to finish.",
+                        )
+                        break
+
+                    self.logger.info(
+                        "Added payload file -> '%s' with index -> %s to runnable queue.",
+                        item["name"] if item["name"] else item["filename"],
+                        item["index"],
+                    )
+
                     # Update the status to "running" in the data frame to prevent re-processing
                     self.payload_items.loc[
                         self.payload_items["name"] == item["name"],
@@ -837,13 +872,14 @@ class PayloadList:
 
     # end method definition
 
-    def run_payload_processing(self) -> None:
+    def run_payload_processing(self, concurrent: int | None = None) -> None:
         """Start the `process_payload_list` method in a daemon thread."""
 
         scheduler_thread = threading.Thread(
             target=self.process_payload_list,
             daemon=True,
             name="Scheduler",
+            kwargs={"concurrent": concurrent},
         )
 
         self.logger.info(
@@ -853,13 +889,6 @@ class PayloadList:
         self._stopped = False
         scheduler_thread.start()
 
-        self.logger.info(
-            "Waiting for thread -> '%s' to complete...",
-            str(scheduler_thread.name),
-        )
-        scheduler_thread.join()
-        self.logger.info("Thread -> '%s' has completed.", str(scheduler_thread.name))
-
     # end method definition
 
     def stop_payload_processing(self) -> None:

pyxecm/customizer/api/common/router.py

@@ -8,7 +8,7 @@ import tempfile
 from http import HTTPStatus
 from typing import Annotated
 
-from fastapi import APIRouter, Depends, HTTPException, Query, Response
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
 from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
 
 from pyxecm.customizer.api.auth.functions import get_authorized_user
@@ -16,21 +16,23 @@ from pyxecm.customizer.api.auth.models import User
 from pyxecm.customizer.api.common.functions import PAYLOAD_LIST, list_files_in_directory
 from pyxecm.customizer.api.common.models import CustomizerStatus
 
-router = APIRouter(tags=["default"])
+router = APIRouter()
 
 logger = logging.getLogger("pyxecm.customizer.api.common")
 
 
 @router.get("/", include_in_schema=False)
-async def redirect_to_api() -> RedirectResponse:
+async def redirect_to_api(request: Request) -> RedirectResponse:
     """Redirect from / to /api.
 
     Returns:
         None
 
     """
-    return RedirectResponse(url="/api")
+    # Construct the new URL by appending /api
+    new_url = f"{request.url.path!s}api"
 
+    return RedirectResponse(url=new_url)
 
 @router.get(path="/status", name="Get Status")
 async def get_status() -> CustomizerStatus:
@@ -74,7 +76,7 @@ def shutdown(user: Annotated[User, Depends(get_authorized_user)]) -> JSONRespons
     return JSONResponse({"status": "shutdown"}, status_code=HTTPStatus.ACCEPTED)
 
 
-@router.get(path="/browser_automations/assets")
+@router.get(path="/browser_automations/assets", tags=["payload"])
 def list_browser_automation_files(
     user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
 ) -> JSONResponse:
@@ -90,7 +92,7 @@ def list_browser_automation_files(
     return JSONResponse(result)
 
 
-@router.get(path="/browser_automations/download")
+@router.get(path="/browser_automations/download", tags=["payload"])
 def get_browser_automation_file(
     user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
     file: Annotated[str, Query(description="File name")],

pyxecm/customizer/api/settings.py

@@ -16,6 +16,12 @@ from pydantic_settings import (
 class CustomizerAPISettings(BaseSettings):
     """Settings for the Customizer API."""
 
+    title: str = Field(default="Customizer API", description="Name of the API Service")
+    description: str = Field(
+        default="API provided by [pyxecm](https://github.com/opentext/pyxecm). The documentation for the payload syntax can be found [here](https://opentext.github.io/pyxecm/payload-syntax/).",
+        description="Descriptive text on the SwaggerUI page.",
+    )
+
     api_key: str | None = Field(
         default=None,
         description="Optional API KEY that can be specified that has access to the Customizer API, bypassing the OTDS authentication.",
@@ -26,6 +32,9 @@ class CustomizerAPISettings(BaseSettings):
     root_path: str = Field(default="/", description="Root path for the Customizer API")
     openapi_url: str = Field(default="/api/openapi.json", description="OpenAPI URL")
 
+    concurrent_payloads: int = Field(
+        default=3, description="Maximum number of concurrent payloads that are executed at the same time."
+    )
     import_payload: bool = Field(default=False)
     payload: str = Field(
         default="/payload/payload.yml.gz.b64",

pyxecm/customizer/api/v1_otcs/router.py

@@ -35,10 +35,15 @@ async def put_otcs_logs(
 
     if "all" in hosts:
         hosts = []
-        for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search"]:
+        for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search", "otcs-da"]:
             try:
-                sts_replicas = k8s_object.get_stateful_set_scale(sts).status.replicas
-                hosts.extend([f"{sts}-{i}" for i in range(sts_replicas)])
+                sts_replicas = k8s_object.get_stateful_set_scale(sts)
+
+                if sts_replicas is None:
+                    logger.debug("Cannot get statefulset {sts}")
+                    continue
+
+                hosts.extend([f"{sts}-{i}" for i in range(sts_replicas.status.replicas)])
             except Exception as e:
                 raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR) from e
 
@@ -118,10 +123,15 @@ async def get_otcs_log_files(
     response = {"status": {host: bool(otcs_logs_lock[host].locked()) for host in otcs_logs_lock}, "files": files}
 
     # Extend response with all hosts
-    for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search"]:
+    for sts in ["otcs-admin", "otcs-frontend", "otcs-backend-search", "otcs-da"]:
         try:
-            sts_replicas = k8s_object.get_stateful_set_scale(sts).status.replicas
-            for i in range(sts_replicas):
+            sts_replicas = k8s_object.get_stateful_set_scale(sts)
+
+            if sts_replicas is None:
+                logger.debug("Cannot get statefulset {sts}")
+                continue
+
+            for i in range(sts_replicas.status.replicas):
                 host = f"{sts}-{i}"
 
                 if host in otcs_logs_lock: