pyxecm 2.0.2__tar.gz → 2.0.4__tar.gz

{pyxecm-2.0.2/pyxecm.egg-info → pyxecm-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyxecm
-Version: 2.0.2
+Version: 2.0.4
 Summary: A Python library to interact with Opentext Extended ECM REST API
 Author-email: Kai Gatzweiler <kgatzweiler@opentext.com>, "Dr. Marc Diefenbruch" <mdiefenb@opentext.com>
 Project-URL: Homepage, https://github.com/opentext/pyxecm
@@ -46,7 +46,7 @@ Requires-Dist: shapely; extra == "dataloader"
 Requires-Dist: cartopy; extra == "dataloader"
 Requires-Dist: psycopg; extra == "dataloader"
 Provides-Extra: sap
-Requires-Dist: pyrfc==2.8.3; extra == "sap"
+Requires-Dist: pyrfc==3.3.1; extra == "sap"
 Provides-Extra: profiling
 Requires-Dist: pyinstrument; extra == "profiling"
 Dynamic: license-file

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyproject.toml

@@ -1,3 +1,4 @@
+
 [build-system]
 build-backend = 'setuptools.build_meta'
 requires = ['setuptools >= 61.0']
@@ -48,7 +49,7 @@ keywords = [
 name = 'pyxecm'
 readme = 'README.md'
 requires-python = '>=3.10'
-version = "2.0.2"
+version = "2.0.4"
 
 [[project.authors]]
 email = 'kgatzweiler@opentext.com'
@@ -68,7 +69,7 @@ dataloader = [
   'cartopy',
   'psycopg',
 ]
-sap = ['pyrfc==2.8.3']
+sap = ['pyrfc==3.3.1']
 profiling = ['pyinstrument']
 
 [project.urls]
@@ -176,8 +177,26 @@ dev = [
   "ruff",
   'python-decouple',
   'settings-doc',
+  "pytest>=8.3.5",
+  "pytest-mock>=3.14.0",
+  "pytest-cov>=6.1.1",
+  "httpx>=0.28.1",
+  "requests-mock>=1.12.1",
 ]
 
 [tool.ruff]
 line-length = 120
 exclude = []
+
+[tool.ruff.lint.extend-per-file-ignores]
+"tests/**/*.py" = [
+    "INP001","D100","D103","ANN",
+    "S101", # asserts allowed in tests...
+    "ARG", # Unused function args -> fixtures nevertheless are functionally relevant...
+    "FBT", # Don't care about booleans as positional arguments in tests, e.g. via @pytest.mark.parametrize()
+    "PLR2004", # Magic value used in comparison, ...
+    "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
+]
+"**/__init__.py" = [
+  "D104"
+]

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/__init__.py

@@ -1,4 +1,4 @@
-"""pyxecm - A python library to interact with Opentext Extended ECM REST API."""
+"""pyxecm - A python library to interact with Opentext REST APIs."""
 
 from .avts import AVTS
 from .coreshare import CoreShare
@@ -8,7 +8,8 @@ from .otca import OTCA
 from .otcs import OTCS
 from .otds import OTDS
 from .otiv import OTIV
+from .otkd import OTKD
 from .otmm import OTMM
 from .otpd import OTPD
 
-__all__ = ["AVTS", "OTAC", "OTAWP", "OTCA", "OTCS", "OTDS", "OTIV", "OTMM", "OTPD", "CoreShare"]
+__all__ = ["AVTS", "OTAC", "OTAWP", "OTCA", "OTCS", "OTDS", "OTIV", "OTKD", "OTMM", "OTPD", "CoreShare"]

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/avts.py

@@ -396,8 +396,10 @@ class AVTS:
 
         if response is not None:
             self._accesstoken = response.get("access_token", None)
+        else:
+            self._accesstoken = None
 
-        return response
+        return self._accesstoken
 
     # end method definition
 

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/coreshare.py

@@ -27,6 +27,7 @@ import time
 import urllib.parse
 from http import HTTPStatus
 from importlib.metadata import version
+from urllib.parse import parse_qs, urlparse
 
 import requests
 
@@ -522,7 +523,7 @@ class CoreShare:
                 Can be used to provide a more specific error message
                 in case an error occurs.
             show_error (bool, optional):
-                True: write an error to the log file
+                True: write an error to the log file (this is the default)
                 False: write a warning to the log file
 
         Returns:
@@ -933,6 +934,71 @@ class CoreShare:
 
     # end method definition
 
+    def get_groups_iterator(
+        self,
+        count: int | None = None,
+    ) -> iter:
+        """Get an iterator object that can be used to traverse all Core Share groups.
+
+        Returning a generator avoids loading a large number of items into memory at once. Instead you
+        can iterate over the potential large list of Core Share groups.
+
+        Example usage:
+            groups = core_share_object.get_groups_iterator(page_size=10)
+            for group in groups:
+                logger.info("Traversing Core Share group -> %s", group["name"])
+
+        Args:
+            count (int | None, optional):
+                The chunk size for the number of groups returned by one
+                REST API call. If None, then a default of 250 is used.
+
+        Returns:
+            iter:
+                A generator yielding one OTDS group per iteration.
+                If the REST API fails, returns no value.
+
+        """
+
+        offset = 0
+
+        while True:
+            response = self.get_groups(
+                offset=offset,
+                count=count,
+            )
+            if not response or not response.get("results", []):
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+
+            # Yield users one at a time:
+            yield from response["results"]
+
+            # See if we have an additional result page.
+            # If not terminate the iterator and return
+            # no value.
+
+            next_page_url = response["_links"].get("next")
+            if not next_page_url:
+                # Don't return None! Plain return is what we need for iterators.
+                # Natural Termination: If the generator does not yield, it behaves
+                # like an empty iterable when used in a loop or converted to a list:
+                return
+            next_page_url = next_page_url.get("href")
+
+            # Extract the query string from the URL
+            query = urlparse(next_page_url).query
+
+            # Parse the query parameters into a dictionary
+            params = parse_qs(query)
+
+            # Get the 'offset' value as an integer (it's a list by default)
+            offset = int(params.get("offset", [0])[0])
+
+    # end method definition
+
     def add_group(
         self,
         group_name: str,
@@ -1276,7 +1342,7 @@ class CoreShare:
             dict | None:
                 Dictionary with the Core Share group data or None if the request fails.
 
-            Example result:
+            Example Response:
             {
                 'results': [
                     {
@@ -1343,15 +1409,15 @@ class CoreShare:
 
     # end method definition
 
-    def get_users(self) -> dict | None:
+    def get_users(self) -> list | None:
         """Get Core Share users.
 
         Args:
             None
 
         Returns:
-            dict | None:
-                Dictionary with the Core Share user data or None if the request fails.
+            list | None:
+                List with the Core Share user data or None if the request fails.
 
             Example response (it is a list!):
             [

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/customizer/api/app.py

@@ -13,7 +13,6 @@ from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from importlib.metadata import version
-from threading import Thread
 
 import uvicorn
 from fastapi import FastAPI
@@ -85,17 +84,11 @@ async def lifespan(
         # Optional Payload
         import_payload(payload_dir=api_settings.payload_dir_optional)
 
-    if api_settings.maintenance_mode:
-        logger.info("Starting maintenance_page thread...")
-        maint_thread = Thread(target=run_maintenance_page, name="maintenance_page")
-        maint_thread.start()
+    logger.info("Starting maintenance_page thread...")
+    run_maintenance_page()
 
     logger.info("Starting processing thread...")
-    thread = Thread(
-        target=PAYLOAD_LIST.run_payload_processing,
-        name="customization_run_api",
-    )
-    thread.start()
+    PAYLOAD_LIST.run_payload_processing(concurrent=api_settings.concurrent_payloads)
 
     yield
     logger.info("Shutdown")
@@ -104,8 +97,10 @@ async def lifespan(
 
 app = FastAPI(
     docs_url="/api",
-    title="Customizer API",
-    openapi_url="/api/openapi.json",
+    title=api_settings.title,
+    description=api_settings.description,
+    openapi_url=api_settings.openapi_url,
+    root_path=api_settings.root_path,
     lifespan=lifespan,
     version=version("pyxecm"),
     openapi_tags=[
@@ -135,7 +130,6 @@ if api_settings.ws_terminal:
 if api_settings.csai:
     app.include_router(router=v1_csai_router)
 
-
 logger = logging.getLogger("CustomizerAPI")
 app.add_middleware(
     CORSMiddleware,

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/customizer/api/auth/functions.py

@@ -5,14 +5,20 @@ from typing import Annotated
 
 import requests
 from fastapi import APIRouter, Depends, HTTPException, status
-from fastapi.security import OAuth2PasswordBearer
+from fastapi.security import APIKeyHeader, OAuth2PasswordBearer
 
 from pyxecm.customizer.api.auth.models import User
 from pyxecm.customizer.api.settings import api_settings
 
 router = APIRouter()
 
-oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+oauth2_scheme = OAuth2PasswordBearer(
+    tokenUrl="token",
+    auto_error=False,
+    description="Authenticate with OTDS, user needs to be member of 'otadmins@otds.admin' group",
+    scheme_name="OTDS Authentication",
+)
+apikey_header = APIKeyHeader(name="x-api-key", auto_error=False, scheme_name="APIKey")
 
 
 def get_groups(response: dict, token: str) -> list:
@@ -42,43 +48,44 @@ def get_groups(response: dict, token: str) -> list:
     return []
 
 
-async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]) -> User:
+async def get_current_user(
+    token: Annotated[str, Depends(oauth2_scheme)], api_key: Annotated[str, Depends(apikey_header)]
+) -> User:
     """Get the current user from OTDS and verify it."""
 
-    if api_settings.api_token is not None and token == api_settings.api_token:
+    if api_settings.api_key is not None and api_key == api_settings.api_key:
         return User(
             id="api",
-            full_name="API Token",
+            full_name="API Key",
             groups=["otadmins@otds.admin"],
             is_admin=True,
             is_sysadmin=True,
         )
 
-    url = api_settings.otds_url + "/otdsws/rest/currentuser"
-    headers = {
-        "Accept": "application/json",
-        "otdsticket": token,
-    }
-    response = requests.request("GET", url, headers=headers, timeout=2)
-
-    if response.ok:
-        response = json.loads(response.text)
-
-        user = User(
-            id=response["user"]["id"],
-            full_name=response["user"]["name"],
-            groups=get_groups(response, token),
-            is_admin=response["isAdmin"],
-            is_sysadmin=response["isSysAdmin"],
-        )
-
-        return user
-    else:
-        raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
-            detail="Invalid authentication credentials",
-            headers={"WWW-Authenticate": "Bearer"},
-        )
+    if token and token.startswith("*OTDSSSO*"):
+        url = api_settings.otds_url + "/otdsws/rest/currentuser"
+        headers = {
+            "Accept": "application/json",
+            "otdsticket": token,
+        }
+        response = requests.request("GET", url, headers=headers, timeout=2)
+
+        if response.ok:
+            response = json.loads(response.text)
+
+            return User(
+                id=response["user"]["id"],
+                full_name=response["user"]["name"],
+                groups=get_groups(response, token),
+                is_admin=response["isAdmin"],
+                is_sysadmin=response["isSysAdmin"],
+            )
+
+    raise HTTPException(
+        status_code=status.HTTP_401_UNAUTHORIZED,
+        detail="Invalid authentication credentials",
+        headers={"WWW-Authenticate": "Bearer"},
+    )
 
 
 async def get_authorized_user(current_user: Annotated[User, Depends(get_current_user)]) -> User:

pyxecm-2.0.4/pyxecm/customizer/api/common/functions.py

@@ -0,0 +1,101 @@
+"""Define common functions."""
+
+import logging
+import os
+
+from pyxecm.customizer.api.common.payload_list import PayloadList
+from pyxecm.customizer.api.settings import CustomizerAPISettings, api_settings
+from pyxecm.customizer.k8s import K8s
+from pyxecm.customizer.settings import Settings
+from pyxecm.otcs import OTCS
+
+logger = logging.getLogger("pyxecm.customizer.api")
+
+# Create a LOCK dict for singleton logs collection
+LOGS_LOCK = {}
+# Initialize the globel Payloadlist object
+PAYLOAD_LIST = PayloadList(logger=logger)
+
+
+def get_k8s_object() -> K8s:
+    """Get an instance of a K8s object.
+
+    Returns:
+        K8s: Return a K8s object
+
+    """
+
+    return K8s(logger=logger, namespace=api_settings.namespace)
+
+
+def get_otcs_object() -> OTCS:
+    """Get an instance of a K8s object.
+
+    Returns:
+        K8s: Return a K8s object
+
+    """
+    settings = Settings()
+
+    cs = OTCS(
+        protocol=settings.otcs.url_backend.scheme,
+        hostname=settings.otcs.url_backend.host,
+        port=settings.otcs.url_backend.port,
+        public_url=str(settings.otcs.url),
+        username=settings.otcs.username,
+        password=settings.otcs.password.get_secret_value(),
+        user_partition=settings.otcs.partition,
+        resource_name=settings.otcs.resource_name,
+        base_path=settings.otcs.base_path,
+        support_path=settings.otcs.support_path,
+        download_dir=settings.otcs.download_dir,
+        feme_uri=settings.otcs.feme_uri,
+        logger=logger,
+    )
+
+    cs.authenticate()
+
+    return cs
+
+
+def get_settings() -> CustomizerAPISettings:
+    """Get the API Settings object.
+
+    Returns:
+        CustomizerPISettings: Returns the API Settings
+
+    """
+
+    return api_settings
+
+
+def get_otcs_logs_lock() -> dict:
+    """Get the Logs LOCK dict.
+
+    Returns:
+        The dict with all LOCKS for the logs
+
+    """
+
+    return LOGS_LOCK
+
+
+def list_files_in_directory(directory: str) -> dict:
+    """Recursively list files in a directory and return a nested JSON structure with URLs."""
+    result = {}
+    for root, dirs, files in os.walk(directory):
+        # Sort directories and files alphabetically
+        dirs.sort()
+        files.sort()
+
+        current_level = result
+        path_parts = root.split(os.sep)
+        relative_path = os.path.relpath(root, directory)
+        for part in path_parts[len(directory.split(os.sep)) :]:
+            if part not in current_level:
+                current_level[part] = {}
+            current_level = current_level[part]
+        for file in files:
+            file_path = os.path.join(relative_path, file)
+            current_level[file] = file_path
+    return result

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/customizer/api/common/payload_list.py

@@ -609,7 +609,7 @@ class PayloadList:
 
         # Log each runnable item
         for _, row in runnable_df.iterrows():
-            self.logger.info(
+            self.logger.debug(
                 "Added payload file -> '%s' with index -> %s to runnable queue.",
                 row["name"] if row["name"] else row["filename"],
                 row["index"],
@@ -619,12 +619,35 @@ class PayloadList:
 
     # end method definition
 
-    def process_payload_list(self) -> None:
+    def pick_running(self) -> int:
+        """Pick all PayloadItems with status "running".
+
+        Returns:
+            pd.DataFrame:
+                A list of running payload items.
+
+        """
+
+        if self.payload_items.empty:
+            return 0
+
+        all_status = self.payload_items["status"].value_counts().to_dict()
+
+        return all_status.get("running", 0)
+
+    # end method definition
+
+    def process_payload_list(self, concurrent: int | None = None) -> None:
         """Process runnable payloads.
 
+        Args:
+            concurrent (int | None, optional):
+                The maximum number of concurrent payloads to run at any given time.
+
         Continuously checks for runnable payload items and starts their
         "process_payload" method in separate threads.
         Runs as a daemon until the customizer ends.
+
         """
 
         def run_and_complete_payload(payload_item: pd.Series) -> None:
@@ -817,6 +840,18 @@ class PayloadList:
             # Start a thread for each runnable item (item is a pd.Series)
             if runnable_items is not None:
                 for _, item in runnable_items.iterrows():
+                    if concurrent and self.pick_running() >= concurrent:
+                        self.logger.debug(
+                            "Reached concurrency limit of %s payloads. Waiting for one to finish.",
+                        )
+                        break
+
+                    self.logger.info(
+                        "Added payload file -> '%s' with index -> %s to runnable queue.",
+                        item["name"] if item["name"] else item["filename"],
+                        item["index"],
+                    )
+
                     # Update the status to "running" in the data frame to prevent re-processing
                     self.payload_items.loc[
                         self.payload_items["name"] == item["name"],
@@ -837,13 +872,14 @@ class PayloadList:
 
     # end method definition
 
-    def run_payload_processing(self) -> None:
+    def run_payload_processing(self, concurrent: int | None = None) -> None:
         """Start the `process_payload_list` method in a daemon thread."""
 
         scheduler_thread = threading.Thread(
             target=self.process_payload_list,
             daemon=True,
             name="Scheduler",
+            kwargs={"concurrent": concurrent},
         )
 
         self.logger.info(
@@ -853,13 +889,6 @@ class PayloadList:
         self._stopped = False
         scheduler_thread.start()
 
-        self.logger.info(
-            "Waiting for thread -> '%s' to complete...",
-            str(scheduler_thread.name),
-        )
-        scheduler_thread.join()
-        self.logger.info("Thread -> '%s' has completed.", str(scheduler_thread.name))
-
     # end method definition
 
     def stop_payload_processing(self) -> None:

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/customizer/api/common/router.py

@@ -1,34 +1,38 @@
 """API Implemenation for the Customizer to start and control the payload processing."""
 
 import logging
+import mimetypes
 import os
 import signal
+import tempfile
 from http import HTTPStatus
 from typing import Annotated
 
-from fastapi import APIRouter, Depends, HTTPException
-from fastapi.responses import JSONResponse, RedirectResponse
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
+from fastapi.responses import FileResponse, JSONResponse, RedirectResponse
 
 from pyxecm.customizer.api.auth.functions import get_authorized_user
 from pyxecm.customizer.api.auth.models import User
-from pyxecm.customizer.api.common.functions import PAYLOAD_LIST
+from pyxecm.customizer.api.common.functions import PAYLOAD_LIST, list_files_in_directory
 from pyxecm.customizer.api.common.models import CustomizerStatus
 
-router = APIRouter(tags=["default"])
+router = APIRouter()
 
 logger = logging.getLogger("pyxecm.customizer.api.common")
 
 
 @router.get("/", include_in_schema=False)
-async def redirect_to_api() -> RedirectResponse:
+async def redirect_to_api(request: Request) -> RedirectResponse:
     """Redirect from / to /api.
 
     Returns:
         None
 
     """
-    return RedirectResponse(url="/api")
+    # Construct the new URL by appending /api
+    new_url = f"{request.url.path!s}api"
 
+    return RedirectResponse(url=new_url)
 
 @router.get(path="/status", name="Get Status")
 async def get_status() -> CustomizerStatus:
@@ -70,3 +74,48 @@ def shutdown(user: Annotated[User, Depends(get_authorized_user)]) -> JSONRespons
     os.kill(os.getpid(), signal.SIGTERM)
 
     return JSONResponse({"status": "shutdown"}, status_code=HTTPStatus.ACCEPTED)
+
+
+@router.get(path="/browser_automations/assets", tags=["payload"])
+def list_browser_automation_files(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+) -> JSONResponse:
+    """List all browser automation files."""
+
+    result = list_files_in_directory(
+        os.path.join(
+            tempfile.gettempdir(),
+            "browser_automations",
+        )
+    )
+
+    return JSONResponse(result)
+
+
+@router.get(path="/browser_automations/download", tags=["payload"])
+def get_browser_automation_file(
+    user: Annotated[User, Depends(get_authorized_user)],  # noqa: ARG001
+    file: Annotated[str, Query(description="File name")],
+) -> FileResponse:
+    """Download the logfile for a specific payload."""
+
+    filename = os.path.join(tempfile.gettempdir(), "browser_automations", file)
+
+    if not os.path.isfile(filename):
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND,
+            detail="File -> '{}' not found".format(filename),
+        )
+
+    media_type, _ = mimetypes.guess_type(filename)
+
+    with open(filename, "rb") as f:
+        content = f.read()
+
+    return Response(
+        content,
+        media_type=media_type,
+        headers={
+            "Content-Disposition": f'attachment; filename="{os.path.basename(filename)}"',
+        },
+    )

{pyxecm-2.0.2 → pyxecm-2.0.4}/pyxecm/customizer/api/settings.py

@@ -16,14 +16,25 @@ from pydantic_settings import (
 class CustomizerAPISettings(BaseSettings):
     """Settings for the Customizer API."""
 
-    api_token: str | None = Field(
+    title: str = Field(default="Customizer API", description="Name of the API Service")
+    description: str = Field(
+        default="API provided by [pyxecm](https://github.com/opentext/pyxecm). The documentation for the payload syntax can be found [here](https://opentext.github.io/pyxecm/payload-syntax/).",
+        description="Descriptive text on the SwaggerUI page.",
+    )
+
+    api_key: str | None = Field(
         default=None,
-        description="Optional token that can be specified that has access to the Customizer API, bypassing the OTDS authentication.",
+        description="Optional API KEY that can be specified that has access to the Customizer API, bypassing the OTDS authentication.",
     )
     bind_address: str = Field(default="0.0.0.0", description="Interface to bind the Customizer API.")  # noqa: S104
     bind_port: int = Field(default=8000, description="Port to bind the Customizer API to")
     workers: int = Field(default=1, description="Number of workers to use for the API BackgroundTasks")
+    root_path: str = Field(default="/", description="Root path for the Customizer API")
+    openapi_url: str = Field(default="/api/openapi.json", description="OpenAPI URL")
 
+    concurrent_payloads: int = Field(
+        default=3, description="Maximum number of concurrent payloads that are executed at the same time."
+    )
     import_payload: bool = Field(default=False)
     payload: str = Field(
         default="/payload/payload.yml.gz.b64",
@@ -58,7 +69,7 @@ class CustomizerAPISettings(BaseSettings):
         description="Namespace to use for otxecm resource lookups",
     )
     maintenance_mode: bool = Field(
-        default=True,
+        default=False,
         description="Automatically enable and disable the maintenance mode during payload deployments.",
     )