pyxecm 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

pyxecm/otcs.py

@@ -70,8 +70,8 @@ REQUEST_DOWNLOAD_HEADERS = {
 }
 
 REQUEST_TIMEOUT = 60
-REQUEST_RETRY_DELAY = 20
-REQUEST_MAX_RETRIES = 2
+REQUEST_RETRY_DELAY = 30
+REQUEST_MAX_RETRIES = 4
 
 default_logger = logging.getLogger(MODULE_NAME)
 
@@ -245,9 +245,11 @@ class OTCS:
         password: str | None = None,
         user_partition: str = "Content Server Members",
         resource_name: str = "cs",
+        resource_id: str = "",
         default_license: str = "X3",
         otds_ticket: str | None = None,
         base_path: str = "/cs/cs",
+        support_path: str = "/cssupport",
         thread_number: int = 3,
         download_dir: str | None = None,
         feme_uri: str | None = None,
@@ -273,7 +275,9 @@ class OTCS:
                 The name of the OTDS partition for OTCS users.
                 Default is "Content Server Members".
             resource_name (str, optional):
-                The name of the OTDS resource for OTCS. Dault is "cs".
+                The name of the OTDS resource for OTCS. Default is "cs".
+            resource_id (str, optional):
+                The ID of the OTDS resource for OTCS. Default is "".
             default_license (str, optional):
                 The name of the default user license. Default is "X3".
             otds_ticket (str, optional):
@@ -282,6 +286,9 @@ class OTCS:
                 The base path segment of the Content Server URL.
                 This typically is /cs/cs on a Linux deployment or /cs/cs.exe
                 on a Windows deployment.
+            support_path (str, optional):
+                The support path of the Content Server. This is typically
+                /cssupport on a Linux deployment. This is also the default.
             thread_number (int, optional):
                 The number of threads for parallel processing for data loads.
             download_dir (str | None, optional):
@@ -347,24 +354,29 @@ class OTCS:
         else:
             otcs_config["resource"] = ""
 
+        if resource_id:
+            otcs_config["resourceId"] = resource_id
+        else:
+            otcs_config["resourceId"] = None
+
         if default_license:
             otcs_config["license"] = default_license
         else:
             otcs_config["license"] = ""
 
-        otcs_config["feme_uri"] = feme_uri
+        otcs_config["femeUri"] = feme_uri
 
         otcs_base_url = protocol + "://" + otcs_config["hostname"]
         if str(port) not in ["80", "443"]:
             otcs_base_url += ":{}".format(port)
         otcs_config["baseUrl"] = otcs_base_url
-        otcs_support_url = otcs_base_url + "/cssupport"
+        otcs_support_url = otcs_base_url + support_path
         otcs_config["supportUrl"] = otcs_support_url
 
         if public_url is None:
             public_url = otcs_base_url
 
-        otcs_public_support_url = public_url + "/cssupport"
+        otcs_public_support_url = public_url + support_path
         otcs_config["supportPublicUrl"] = otcs_public_support_url
 
         otcs_config["configuredUrl"] = otcs_support_url + "/csconfigured"
@@ -639,6 +651,58 @@ class OTCS:
 
     # end method definition
 
+    def partition_name(self) -> str:
+        """Return the OTDS user partition for Content Server.
+
+        Returns:
+            str:
+                The Content Server OTDS user partition.
+
+        """
+
+        return self.config()["partition"]
+
+    # end method definition
+
+    def resource_name(self) -> str:
+        """Return the OTDS resource name of Content Server.
+
+        Returns:
+            str:
+                The Content Server OTDS resource name.
+
+        """
+
+        return self.config()["resource"]
+
+    # end method definition
+
+    def resource_id(self) -> str:
+        """Return the OTDS resource ID of Content Server.
+
+        Returns:
+            str:
+                The Content Server OTDS resource ID.
+
+        """
+
+        return self.config()["resourceId"]
+
+    # end method definition
+
+    def set_resource_id(self, resource_id: str) -> None:
+        """Set the OTDS resource ID of Content Server.
+
+        Args:
+            resource_id (str):
+                The Content Server OTDS resource ID.
+
+        """
+
+        self.config()["resourceId"] = resource_id
+
+    # end method definition
+
     def get_data(self) -> Data:
         """Get the Data object that holds all loaded Content Server items (see method load_items()).
 
@@ -925,7 +989,19 @@ class OTCS:
                         str(REQUEST_RETRY_DELAY),
                     )
                     retries += 1
-                    time.sleep(REQUEST_RETRY_DELAY)  # Add a delay before retrying
+
+                    if not self.is_ready():
+                        self.logger.warning(
+                            "Content Server is not ready to receive requests. Waiting for state change in %s seconds...",
+                            str(REQUEST_RETRY_DELAY),
+                        )
+
+                        while not self.is_ready():
+                            time.sleep(REQUEST_RETRY_DELAY)  # Add a delay before retrying
+
+                    else:
+                        time.sleep(REQUEST_RETRY_DELAY)  # Add a delay before retrying
+
                 else:
                     self.logger.error(
                         "%s; connection error",
@@ -1116,7 +1192,8 @@ class OTCS:
                 The name of the substructure that includes the values.
 
         Returns:
-            bool: True if the value was found, False otherwise
+            bool:
+                True if the value was found, False otherwise
 
         """
 
@@ -1581,10 +1658,9 @@ class OTCS:
         otcs_ticket = None
 
         if wait_for_ready:
-            self.logger.info("Check if OTCS is ready...")
             while not self.is_ready():
                 self.logger.debug(
-                    "OTCS is not ready to receive requests yet. Waiting additional 30 seconds...",
+                    "OTCS is not ready to receive requests yet. Waiting 30 seconds...",
                 )
                 time.sleep(30)
 
@@ -3833,7 +3909,7 @@ class OTCS:
         nickname: str,
         show_error: bool = False,
     ) -> dict | None:
-        """Assign a nickname to an Extended ECM node (e.g. workspace).
+        """Assign a nickname to an OTCS node (e.g. workspace).
 
         Some naming conventions for the nickname are automatically applied:
         - replace "-" with "_"
@@ -6672,7 +6748,7 @@ class OTCS:
             request_url,
         )
 
-        return self.do_request(
+        response = self.do_request(
             url=request_url,
             method="POST",
             headers=request_header,
@@ -6682,6 +6758,25 @@ class OTCS:
             ),
         )
 
+        try:
+            if response["results"]["data"]["status"]["error_count"] > 0:
+                self.logger.error("Error occoured during package deployment")
+            else:
+                self.logger.info(
+                    "Transport successfully deployed %s items",
+                    response["results"]["data"]["status"]["success_count"],
+                )
+
+            for error in response["results"]["data"]["status"]["errors"]:
+                self.logger.error(
+                    "Transport deployment error: %s (%s) -> %s", error["name"], error["id"], error["error"]
+                )
+
+        except Exception as e:
+            self.logger.debug(e)
+
+        return response
+
     # end method definition
 
     def deploy_transport(
@@ -14714,7 +14809,29 @@ class OTCS:
             row (dict):
                 The row data to extend with keys for each attribute.
             categories (dict):
-                The categories of the node.
+                The categories of the node. This is a structure like
+                {
+                    "links" = {}
+                    "results" = [
+                        {
+                            "data" = {
+                                "categories" = {
+                                    "14885_10_1_11 = "Aerospace", # Multi-value set row 1, attribute 11
+                                    "14885_10_2_11 = "Automotive" # Multi-value set row 2, attribute 11
+                                    "14885_14" = ["Content"] # Multi-value attribute
+                                    "14885_15" = "Test" # Single value attribute
+                                }
+                            }
+                            "metadata" = {
+                                "categories" = {
+                                    "14885_10" = {...} # Definition of the set
+                                    "14885_10_x_11" = {...} # Definition of the set attribute
+                                    "144885_14" = {...} # Definition of multi-value attribute
+                                }
+                            }
+                        }
+                    ]
+                }
             prefix (str):
                 The prefix string. Either "workspace_" or "item_" to
                 differentiate attributes on workspace level and attributes
@@ -14726,6 +14843,19 @@ class OTCS:
 
         """
 
+        def get_attribute_identifier(s: str) -> str:
+            # Get the prefix of two numbers separated by an underscore:
+            match = re.match(r"^([^_]+_[^_]+)", s)
+            return match.group(1) if match else ""
+
+        def get_column_identifier(s: str) -> str:
+            # Cut out the third number if there's a third number. Used for set attributes
+            match = re.match(r"^([^_]+_[^_]+)(?:.*_([0-9]+))?$", s)
+            return f"{match.group(1)}_{match.group(2)}" if match and match.group(2) else match.group(1)
+
+        def get_set_identifier(s: str) -> str:
+            return re.sub(r"^([^_]+_[^_]+_)[^_]+", r"\1x", s)
+
         if not categories or "results" not in categories:
             return False
 
@@ -14739,15 +14869,37 @@ class OTCS:
             # Replace non-aphanumeric characters in the name with underscores
             # but avoid having multiple underscores following each other:
             category_name = re.sub(r"[^a-z0-9]+", "_", category_name.lower())
+
+            # Iterate over all attributes. For multi-value sets (which are basically a matrix or table)
+            # we do a special handling. The values of each column of such a table are written as a value list in a
+            # separate data frame column. So we slice the multi-value set by columns (not rows).
+            # This way this can later on recombined by "columns_to_add_table" in the payload.
             for key in attributes:
                 value = attributes[key]
+                # We don't want "_x_" in the column identifiers as we create a list:
+                column_key = get_column_identifier(key)
+                # The attribute key should just be the category ID (before first "_")
+                # and the attribute or set ID (after first "_"):
+                attribute_key = get_attribute_identifier(key)
+                meta = metadata[attribute_key]
                 if self._use_numeric_category_identifier:  # this value is set be the class initializer
-                    column_header = prefix + key
+                    column_header = prefix + column_key
                 else:
-                    # Construct the final coolumn name by replacing the leading <cat_num>_ with the
+                    # Construct the final column name by replacing the leading <cat_num>_ with the
                     # normalized name of the category.
-                    column_header = prefix + re.sub(r"^[^_]+_", category_name + "_", key)
-                row[column_header] = value
+                    column_header = prefix + re.sub(r"^[^_]+_", category_name + "_", column_key)
+                # Check if meta is the schema for a multi-value set. This is the
+                # case if "multi_value" is True _and_ "persona" is "set":
+                if meta.get("multi_value", False) and meta.get("persona") == "set":
+                    # Is it the first value line? Then we need to initialize the list...
+                    if column_header not in row:
+                        row[column_header] = []
+                    row[column_header].append(value)
+                else:
+                    # Not a multi-value set. We just write the value
+                    # into the data frame (in case it is a multi-value
+                    # a)
+                    row[column_header] = value
 
         return True
 
@@ -14889,7 +15041,7 @@ class OTCS:
                         # We try to avoid calculating the node categories more than once
                         # by doing it here and use it for filtering _and_ for
                         # data frame columns. We only need the category metadata if we
-                        # have category/attribute filters:
+                        # have category/attribute filters or if we want columns with attributze names instead of IDs:
                         if workspace_metadata or filter_workspace_category or filter_workspace_attributes:
                             categories = self.get_node_categories(
                                 node_id=subnode["id"],
@@ -15172,7 +15324,7 @@ class OTCS:
 
     # end method definition
 
-    def feme_embedd_metadata(
+    def aviator_embed_metadata(
         self,
         node_id: int,
         node: dict | None = None,
@@ -15186,7 +15338,7 @@ class OTCS:
         workspace_metadata: bool = True,
         remove_existing: bool = False,
     ) -> None:
-        """Run FEME metadata embedding on provided node for Content Aviator.
+        """Run Content Aviator metadata embedding on provided node with FEME tool.
 
         Args:
             node_id (int):
@@ -15215,9 +15367,11 @@ class OTCS:
 
         """
 
+        success = True
+
         async def _inner(
             uri: str,
-            node: dict,
+            node_properties: dict,
             crawl: bool,
             wait_for_completion: bool,
             message_override: dict | None,
@@ -15227,7 +15381,11 @@ class OTCS:
             image_prompt: str = "",
             workspace_metadata: bool = True,
             remove_existing: bool = False,
-        ) -> None:
+        ) -> bool:
+            # This is important as the sub-method needs to write
+            # to the 'success' variable:
+            nonlocal success
+
             self.logger.debug("Open WebSocket connection to -> %s", uri)
             async with websockets.connect(uri) as websocket:
                 # Define if one node (index), or all childs should be processed (crawl)
@@ -15235,7 +15393,7 @@ class OTCS:
 
                 message = {
                     "task": task,  # either "index" or "crawl". "crawl" means traversing OTCS workspaces and folders.
-                    "nodes": [node],  # the list of (root) node IDs to process
+                    "nodes": [node_properties],  # the list of (root) nodes to process
                     "documents": document_metadata,  # process metadata of documents
                     "workspaces": workspace_metadata,  # process metadata of workspaces
                     "images": images,  # enable image processing via LLM (Gemini) - just content of images
@@ -15249,11 +15407,11 @@ class OTCS:
                 }
                 if message_override:
                     message.update(message_override)
-                self.logger.debug(
-                    "Start FEME on -> %s (%s), type -> %s, crawl -> %s, wait for completion -> %s, workspaces -> %s, documents -> %s, images -> %s",
-                    node["name"],
-                    node["id"],
-                    node["type"],
+                self.logger.info(
+                    "Start Content Aviator embedding on -> '%s' (%s), type -> %s, crawl -> %s, wait for completion -> %s, workspaces -> %s, documents -> %s, images -> %s",
+                    node_properties["name"],
+                    node_properties["id"],
+                    node_properties["type"],
                     crawl,
                     wait_for_completion,
                     workspace_metadata,
@@ -15275,6 +15433,7 @@ class OTCS:
                             "Timeout Error during FEME WebSocket connection, WebSocket did not receive a message in time (%ss)",
                             timeout,
                         )
+                        success = False
                         break
 
                     self.logger.debug("Received WebSocket response -> %s", response)
@@ -15319,20 +15478,20 @@ class OTCS:
                 "Cannot get node with ID -> %s, skipping FEME embedding!",
                 node_id,
             )
-            return
+            return False
         try:
-            node_data = node["results"]["data"]["properties"]
-        except json.JSONDecodeError:
+            node_properties = node["results"]["data"]["properties"] if "results" in node else node["data"]["properties"]
+        except (json.JSONDecodeError, KeyError):
             self.logger.error(
-                "Cannot decode data for node with ID -> %s, skipping FEME embedding.",
+                "Cannot decode data for node with ID -> %s, skipping embedding with FEME.",
                 node_id,
             )
-            return
+            return False
 
-        uri = self._config["feme_uri"]
+        uri = self._config["femeUri"]
         task = _inner(
             uri=uri,
-            node=node_data,
+            node_properties=node_properties,
             crawl=crawl,
             wait_for_completion=wait_for_completion,
             message_override=message_override,
@@ -15348,16 +15507,21 @@ class OTCS:
             event_loop.run_until_complete(task)
         except websockets.exceptions.ConnectionClosed:  # :
             self.logger.error("WebSocket connection was closed!")
+            success = False
 
         except TimeoutError:
             self.logger.error(
                 "Timeout error during FEME WebSocket connection, WebSocket did not receive a message in time (%ss)",
                 timeout,
             )
+            success = False
 
-        except Exception:
-            self.logger.error("Error during FEME WebSocket connection!")
+        except Exception as exc:
+            self.logger.error("Error during FEME WebSocket connection! -> %s", exc)
+            success = False
 
         event_loop.close()
 
+        return success
+
     # end method definition

pyxecm/otds.py

@@ -81,6 +81,7 @@ class OTDS:
         password: str | None = None,
         otds_ticket: str | None = None,
         bind_password: str | None = None,
+        admin_partition: str = "otds.admin",
         logger: logging.Logger = default_logger,
     ) -> None:
         """Initialize the OTDS object.
@@ -99,6 +100,8 @@ class OTDS:
             otds_ticket (str | None, optional):
                 Authentication ticket of OTDS.
             bind_password (str | None, optional): TODO
+            admin_partition (str, optional):
+                Name of the admin partition. Default is "otds.admin".
             logger (logging.Logger, optional):
                 The logging object to use for all log messages. Defaults to default_logger.
 
@@ -142,6 +145,8 @@ class OTDS:
         else:
             otds_config["bindPassword"] = ""
 
+        otds_config["adminPartition"] = admin_partition
+
         if otds_ticket:
             self._cookie = {"OTDSTicket": otds_ticket}
 
@@ -443,6 +448,19 @@ class OTDS:
 
     # end method definition
 
+    def admin_partition_name(self) -> str:
+        """Return OTDS admin partition name.
+
+        Returns:
+            str:
+                The OTDS admin partition name.
+
+        """
+
+        return self.config()["adminPartition"]
+
+    # end method definition
+
     def do_request(
         self,
         url: str,
@@ -544,7 +562,7 @@ class OTDS:
                         in response.text  # OTDS seems to return 400 and not 401 for token expiry (in some cases like impersonation)
                     )
                 ):
-                    self.logger.debug("Session has expired - try to re-authenticate...")
+                    self.logger.info("Session has expired - try to re-authenticate...")
                     self.authenticate(revalidate=True)
                     retries += 1
                 else:
@@ -619,7 +637,7 @@ class OTDS:
                     else:
                         return None
             # end try
-            self.logger.debug(
+            self.logger.info(
                 "Retrying REST API %s call -> %s... (retry = %s, cookie -> %s)",
                 method,
                 url,
@@ -2648,17 +2666,19 @@ class OTDS:
             if existing_license:
                 request_url += "/" + existing_license[0]["id"]
             else:
-                self.logger.debug(
-                    "No existing license found for resource -> '%s' - adding a new license...",
+                self.logger.info(
+                    "No existing license found for product -> '%s' and resource -> '%s' - adding a new license...",
+                    product_name,
                     resource_id,
                 )
                 # change strategy to create a new license:
                 update = False
 
         self.logger.debug(
-            "Adding product license -> '%s' for product -> '%s' to resource ->'%s'; calling -> %s",
+            "%s product license -> '%s' for product -> '%s' to resource ->'%s'; calling -> %s",
+            "Adding" if not update else "Updating",
             path_to_license_file,
-            product_description,
+            "{} ({})".format(product_name, product_description) if product_description else product_name,
             resource_id,
             request_url,
         )
@@ -2670,9 +2690,10 @@ class OTDS:
                 method="PUT",
                 json_data=license_post_body_json,
                 timeout=None,
-                failure_message="Failed to update product license -> '{}' for product -> '{}'".format(
+                failure_message="Failed to update product license -> '{}' for product -> '{}' to resource -> '{}'".format(
                     path_to_license_file,
-                    product_description,
+                    "{} ({})".format(product_name, product_description) if product_description else product_name,
+                    resource_id,
                 ),
             )
         else:
@@ -2682,9 +2703,10 @@ class OTDS:
                 method="POST",
                 json_data=license_post_body_json,
                 timeout=None,
-                failure_message="Failed to add product license -> '{}' for product -> '{}'".format(
+                failure_message="Failed to add product license -> '{}' for product -> '{}' to resource -> '{}'".format(
                     path_to_license_file,
-                    product_description,
+                    "{} ({})".format(product_name, product_description) if product_description else product_name,
+                    resource_id,
                 ),
             )
 
@@ -3871,17 +3893,17 @@ class OTDS:
         """
 
         encoded_client_secret = "{}:{}".format(client_id, client_secret).encode("utf-8")
-        access_token_request_headers = {
+
+        request_header = {
             "Authorization": "Basic " + base64.b64encode(encoded_client_secret).decode("utf-8"),
             "Content-Type": "application/x-www-form-urlencoded",
         }
-
         request_url = self.token_url()
 
         response = requests.post(
             url=request_url,
             data={"grant_type": "client_credentials"},
-            headers=access_token_request_headers,
+            headers=request_header,
             timeout=REQUEST_TIMEOUT,
         )
 

{pyxecm-2.0.0.dist-info → pyxecm-2.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyxecm
-Version: 2.0.0
+Version: 2.0.1
 Summary: A Python library to interact with Opentext Extended ECM REST API
 Author-email: Kai Gatzweiler <kgatzweiler@opentext.com>, "Dr. Marc Diefenbruch" <mdiefenb@opentext.com>
 Project-URL: Homepage, https://github.com/opentext/pyxecm
@@ -28,16 +28,17 @@ Requires-Dist: pandas
 Requires-Dist: python-magic
 Requires-Dist: websockets
 Requires-Dist: pydantic-settings
-Requires-Dist: fastapi
+Requires-Dist: fastapi>=0.115.12
 Requires-Dist: uvicorn
 Requires-Dist: python-multipart
 Requires-Dist: aiofiles
 Requires-Dist: asyncio
 Requires-Dist: jinja2
 Requires-Dist: prometheus-fastapi-instrumentator
+Requires-Dist: psycopg[binary,pool]>=3.2.6
 Provides-Extra: browserautomation
-Requires-Dist: selenium; extra == "browserautomation"
 Requires-Dist: chromedriver_autoinstaller; extra == "browserautomation"
+Requires-Dist: playwright>=1.52.0; extra == "browserautomation"
 Provides-Extra: dataloader
 Requires-Dist: pandas; extra == "dataloader"
 Requires-Dist: pyyaml; extra == "dataloader"
@@ -45,6 +46,7 @@ Requires-Dist: python-hcl2; extra == "dataloader"
 Requires-Dist: tropycal; extra == "dataloader"
 Requires-Dist: shapely; extra == "dataloader"
 Requires-Dist: cartopy; extra == "dataloader"
+Requires-Dist: psycopg; extra == "dataloader"
 Provides-Extra: sap
 Requires-Dist: pyrfc==2.8.3; extra == "sap"
 Provides-Extra: profiling
@@ -77,21 +79,6 @@ Create an `.env` file as described here: [sample-environment-variables](customiz
 python -m pyxecm.customizer.api
 ```
 
-??? example "Sample Output"
-    ```console
-    INFO:     Started server process [93861]
-    INFO:     Waiting for application startup.
-    31-Mar-2025 12:49:53 INFO [CustomizerAPI] [MainThread] Starting maintenance_page thread...
-    31-Mar-2025 12:49:53 INFO [CustomizerAPI] [MainThread] Starting processing thread...
-    31-Mar-2025 12:49:53 INFO [CustomizerAPI.payload_list] [customization_run_api] Starting 'Scheduler' thread for payload list processing...
-    INFO:     Application startup complete.
-    31-Mar-2025 12:49:53 INFO [CustomizerAPI.payload_list] [customization_run_api] Waiting for thread -> 'Scheduler' to complete...
-    INFO:     Started server process [93861]
-    INFO:     Waiting for application startup.
-    INFO:     Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)
-    INFO:     Application startup complete.
-    INFO:     Uvicorn running on http://0.0.0.0:5555 (Press CTRL+C to quit)
-    ```
 
 Access the Customizer API at [http://localhost:8000/api](http://localhost:8000/api)
 
@@ -117,17 +104,7 @@ nodes = otcs_object.get_subnodes(2000)
 for node in nodes["results"]:
     print(node["data"]["properties"]["id"], node["data"]["properties"]["name"])
 ```
-??? example "Sample Output"
-    ```console
-    13050 Administration
-    13064 Case Management
-    18565 Contract Management
-    18599 Customer Support
-    18536 Engineering & Construction
-    13107 Enterprise Asset Management
-    18632 Human Resources
-    6554 Inbox Folders
-    ```
+
 
 # Disclaimer