pyxecm 0.0.18__py3-none-any.whl → 0.0.19__py3-none-any.whl

pyxecm/__init__.py

@@ -2,16 +2,16 @@ import logging
 import os
 
 # pyxecm packages
-from pyxecm.main import *
-from pyxecm.k8s import *
-from pyxecm.otac import *
-from pyxecm.otcs import *
-from pyxecm.otds import *
-from pyxecm.otiv import *
-from pyxecm.otpd import *
-from pyxecm.payload import *
-from pyxecm.translate import *
-from pyxecm.web import *
+from .main import *
+from .k8s import *
+from .otac import *
+from .otcs import *
+from .otds import *
+from .otiv import *
+from .otpd import *
+from .payload import *
+from .translate import *
+from .web import *
 
 logging.basicConfig(
     format="%(asctime)s %(levelname)s [%(name)s] %(message)s",

pyxecm/assoc.py

@@ -0,0 +1,139 @@
+"""
+Extended ECM Assoc Module to implement functions to read / write from
+so called "Assoc" data structures in Extended ECM. Right now this module
+is used to tweak settings in XML-based transport packages that include
+Assoc structures inside some of the XML elements.
+
+Class: Assoc
+Methods:
+
+stringToDict: convert an Assoc string to an Python dict representing the assoc values
+dictToString: converting an Assoc dict to an Assoc string
+"""
+
+__author__ = "Dr. Marc Diefenbruch"
+__copyright__ = "Copyright 2023, OpenText"
+__credits__ = ["Kai-Philip Gatzweiler"]
+__maintainer__ = "Dr. Marc Diefenbruch"
+__email__ = "mdiefenb@opentext.com"
+
+import re
+import html
+
+
+class Assoc:
+    @classmethod
+    def isUnicodeEscaped(cls, assoc_string: str) -> bool:
+        pattern = r"\\u[0-9a-fA-F]{4}"
+        matches = re.findall(pattern, assoc_string)
+        return len(matches) > 0
+
+
+    @classmethod
+    def escapeUnicode(cls, assoc_string: str) -> str:
+        encoded_string = assoc_string.encode('unicode_escape') # .decode()
+
+        return encoded_string
+
+
+    @classmethod
+    def unescapeUnicode(cls, assoc_string: str) -> str:
+        try:
+            decoded_string = bytes(assoc_string, "utf-8").decode("unicode_escape")
+            return decoded_string
+        except UnicodeDecodeError:
+            return assoc_string
+
+    @classmethod
+    def isHTMLEscaped(cls, assoc_string: str) -> bool:
+        decoded_string = html.unescape(assoc_string)
+        return assoc_string != decoded_string
+
+    @classmethod
+    def unescapeHTML(cls, assoc_string: str) -> str:
+        decoded_string = html.unescape(assoc_string)
+        return decoded_string
+
+    @classmethod
+    def stringToDict(cls, assoc_string: str) -> dict:
+        if cls.isHTMLEscaped(assoc_string):
+            assoc_string = cls.unescapeHTML(assoc_string)
+        if cls.isUnicodeEscaped(assoc_string):
+            assoc_string = cls.unescapeUnicode(assoc_string)
+
+        # Split the string using regex pattern
+        pieces = re.split(r",(?=(?:[^']*'[^']*')*[^']*$)", assoc_string)
+
+        # Trim any leading/trailing spaces from each piece
+        pieces = [piece.strip() for piece in pieces]
+
+        # Split the last pieces from the assoc close tag
+        last_piece = pieces[-1].split(">")[0]
+
+        # Remove the first two and last pieces from the list
+        # the first two are mostly "1" and "?"
+        pieces = pieces[2:-1]
+
+        # Insert the last pieces separately
+        pieces.append(last_piece)
+
+        assoc_dict: dict = {}
+
+        for piece in pieces:
+            name = piece.split("=")[0]
+            if name[0] == "'":
+                name = name[1:]
+            if name[-1] == "'":
+                name = name[:-1]
+            value = piece.split("=")[1]
+            if value[0] == "'":
+                value = value[1:]
+            if value[-1] == "'":
+                value = value[:-1]
+            assoc_dict[name] = value
+
+        return assoc_dict
+
+    @classmethod
+    def dictToString(cls, assoc_dict: dict) -> str:
+        assoc_string: str = "A<1,?,"
+
+        for item in assoc_dict.items():
+            assoc_string += "\u0027" + item[0] + "\u0027"
+            assoc_string += "="
+            # Extended ECM's XML is a bit special in cases.
+            # If the value is empty set (curly braces) it does
+            # not put it in quotes. As Extended ECM is also very
+            # picky about XML syntax we better produce it exactly like that.
+            if item[1] == "{}":
+                assoc_string += item[1] + ","
+            else:
+                assoc_string += "\u0027" + item[1] + "\u0027,"
+
+        if assoc_dict.items():
+            assoc_string = assoc_string[:-1]
+        assoc_string += ">"
+        return assoc_string
+
+    @classmethod
+    def extractSubstring(
+        cls, input_string: str, start_sequence: str, stop_sequence: str
+    ):
+        start_index = input_string.find(start_sequence)
+        if start_index == -1:
+            return None
+
+        end_index = input_string.find(stop_sequence, start_index)
+        if end_index == -1:
+            return None
+
+        end_index += len(stop_sequence)
+        return input_string[start_index:end_index]
+
+    @classmethod
+    def extractAssocString(cls, input_string: str, is_escaped: bool = False) -> str:
+        if is_escaped:
+            assoc_string = cls.extractSubstring(input_string, "A<", ">")
+        else:
+            assoc_string = cls.extractSubstring(input_string, "A<", ">")
+        return assoc_string

pyxecm/m365.py

@@ -200,7 +200,7 @@ class M365(object):
             response_object (object): this is reponse object delivered by the request call
             additional_error_message (string, optional): use a more specific error message
                                                          in case of an error
-            show_error (boolean): True: write an error to the log file 
+            show_error (boolean): True: write an error to the log file
                                   False: write a warning to the log file
         Returns:
             dictionary: response information or None in case of an error
@@ -496,7 +496,7 @@ class M365(object):
 
     # end method definition
 
-    def getGroups(self):
+    def getGroups(self, max_number: int = 250):
         """Get list all all groups in M365 tenant
 
         Args:
@@ -512,7 +512,9 @@ class M365(object):
 
         retries = 0
         while True:
-            response = requests.get(request_url, headers=request_header)
+            response = requests.get(
+                request_url, headers=request_header, params={"$top": str(max_number)}
+            )
             if response.ok:
                 return self.parseRequestResponse(response)
             # Check if Session has expired - then re-authenticate and try once more
@@ -990,10 +992,15 @@ class M365(object):
         """
 
         # Get list of all existing M365 groups/teams:
-        response = self.getGroups()
+        response = self.getGroups(max_number=500)
         if not "value" in response or not response["value"]:
             return False
         groups = response["value"]
+        logger.info(
+            "Found -> {} existing M365 groups. Checking which ones should be deleted...".format(
+                len(groups)
+            )
+        )
 
         # Process all groups and check if the< should be
         # deleted:

pyxecm/main.py

@@ -53,8 +53,6 @@ otcs_replicas_backend = 0
 
 # global Logging LEVEL environment variable
 # This produces debug-level output in pod logging
-# and also activates stop_on_error parameter
-# in processTransportPackages()
 LOGLEVEL = os.environ.get("LOGLEVEL", "INFO")
 
 # The following CUST artifacts are created by the main.tf in the python module:
@@ -95,6 +93,7 @@ OTCS_PUBLIC_URL = os.environ.get("OTCS_PUBLIC_URL", "otcs.xecm.dev")
 OTCS_PORT = OTCS_PORT_BACKEND = os.environ.get("OTCS_SERVICE_PORT_OTCS", 8080)
 OTCS_PORT_FRONTEND = 80
 OTCS_ADMIN = os.environ.get("OTCS_ADMIN", "admin")
+OTCS_PASSWORD = os.environ.get("OTCS_PASSWORD", "Opentext1!")
 OTCS_PARTITION = os.environ.get("OTCS_PARTITION", "Content Server Members")
 OTCS_RESOURCE_NAME = "cs"
 OTCS_K8S_STATEFUL_SET_FRONTEND = "otcs-frontend"
@@ -108,7 +107,6 @@ OTCS_LICENSE_FEATURE = "X3"
 # K8s service name for maintenance pod
 OTCS_MAINTENANCE_SERVICE_NAME = "maintenance"
 OTCS_MAINTENANCE_SERVICE_PORT = 80  # K8s service name for maintenance pod
-OTCS_PASSWORD = os.environ.get("OTCS_PASSWORD", "Opentext1!")
 
 # Archive Center constants:
 OTAC_ENABLED = os.environ.get("OTAC_ENABLED", True)
@@ -476,6 +474,9 @@ def initOTCS(
 
         otds_object.updateResource(name="cs", resource=otcs_resource)
 
+    # Allow impersonation of the resource for all users:
+    otds_object.impersonateResource(resource_name)
+
     return otcs_object
 
     # end function definition
@@ -835,7 +836,7 @@ def initOTAWP(otds_object: object, k8s_object: object):
 
     logger.info("OTDS resource ID for AppWorks Platform -> {}".format(awp_resource_id))
 
-    placeholder_values["OTAWP_RESSOURCE_ID"] = str(awp_resource_id)
+    placeholder_values["OTAWP_RESOURCE_ID"] = str(awp_resource_id)
 
     logger.debug("Placeholder values after OTAWP init = {}".format(placeholder_values))
 
@@ -933,6 +934,8 @@ def restartOTCSPods(otcs_object: object, k8s_object: object):
     global otcs_replicas_frontend
     global otcs_replicas_backend
 
+    logger.info("Restart OTCS frontend and backend pods...")
+
     # Restart all frontends:
     for x in range(0, otcs_replicas_frontend):
         pod_name = OTCS_K8S_STATEFUL_SET_FRONTEND + "-" + str(x)
@@ -982,6 +985,8 @@ def restartOTCSPods(otcs_object: object, k8s_object: object):
         logger.info("Reactivate Liveness probe for pod -> {}".format(pod_name))
         k8s_object.execPodCommand(pod_name, ["/bin/sh", "-c", "rm /tmp/keepalive"])
 
+    logger.info("Restart OTCS frontend and backend pods has been completed.")
+
     # end function definition
 
 
@@ -991,7 +996,7 @@ def restartOTACPod(k8s_object: object) -> bool:
     Args:
         k8s_object (object): Kubernetes object
     Returns:
-        boolean: True if restart was done, False if error occured 
+        boolean: True if restart was done, False if error occured
     """
 
     if not OTAC_ENABLED:
@@ -1135,7 +1140,7 @@ def customization_run():
         # Configure required OTDS resources as AppWorks doesn't do this on its own:
         initOTAWP(otds_object, k8s_object)
     else:
-        placeholder_values["OTAWP_RESSOURCE_ID"] = ""
+        placeholder_values["OTAWP_RESOURCE_ID"] = ""
 
     if O365_ENABLED == "true":  # is M365 enabled?
         logger.info("======== Initialize MS Graph API ========")
@@ -1251,17 +1256,18 @@ def customization_run():
         totalStartTime = datetime.now()
 
         payload_object = payload.Payload(
-            cust_payload,
-            CUST_SETTINGS_DIR,
-            k8s_object,
-            otds_object,
-            otac_object,
-            otcs_backend_object,
-            otcs_frontend_object,
-            otiv_object,
-            m365_object,
-            placeholder_values,
-            True if LOGLEVEL == "DEBUG" else False,
+            payload_source=cust_payload,
+            custom_settings_dir=CUST_SETTINGS_DIR,
+            k8s_object=k8s_object,
+            otds_object=otds_object,
+            otac_object=otac_object,
+            otcs_backend_object=otcs_backend_object,
+            otcs_frontend_object=otcs_frontend_object,
+            otcs_restart_callback=restartOTCSPods,
+            otiv_object=otiv_object,
+            m365_object=m365_object,
+            placeholder_values=placeholder_values, # this dict includes placeholder replacements for the Ressource IDs of OTAWP and OTCS
+            stop_on_error=True if LOGLEVEL == "DEBUG" else False,
         )
         # Load the payload file and initialize the payload sections:
         if not payload_object.initPayload():
@@ -1332,11 +1338,11 @@ def customization_run():
         logger.info("OTCS frontend is now back in Production Mode!")
 
     # Restart OTCS frontend and backend pods:
-    logger.info("Restart OTCS frontend and backend pods...")
-    restartOTCSPods(otcs_backend_object, k8s_object)
-    # give some additional time to make sure service is responsive
-    time.sleep(30)
-    logger.info("Restart OTCS frontend and backend pods has been completed.")
+    # logger.info("Restart OTCS frontend and backend pods...")
+    # restartOTCSPods(otcs_backend_object, k8s_object)
+    # # give some additional time to make sure service is responsive
+    # time.sleep(30)
+    # logger.info("Restart OTCS frontend and backend pods has been completed.")
 
     # Restart AppWorksPlatform pod if it is deployed (to make settings effective):
     if OTAWP_ENABLED == "true":  # is AppWorks Platform deployed?

pyxecm/otcs.py

@@ -74,8 +74,6 @@ unpackTransportPackage: Unpack an existing Transport Package into an existing Wo
 deployWorkbench: Deploy an existing Workbench
 deployTransport: Main method to deploy a transport. This uses subfunctions to upload,
                  unpackage and deploy the transport, and creates the required workbench
-replaceInXmlFiles: Replace all occurrences of the search pattern with the replace string in all
-                   XML files in the directory and its subdirectories.
 replaceTransportPlaceholders: Search and replace strings in the XML files of the transport packlage
 
 getWorkspaceTypes: Get all workspace types configured in Extended ECM
@@ -154,7 +152,7 @@ import json
 import urllib.parse
 from datetime import datetime
 import zipfile
-import re
+from pyxecm.xml import *
 
 logger = logging.getLogger(os.path.basename(__file__))
 
@@ -1706,7 +1704,11 @@ class OTCS(object):
     # end method definition
 
     def getNodeByParentAndName(
-        self, parent_id: int, name: str, show_error: bool = False
+        self,
+        parent_id: int,
+        name: str,
+        fields: str = "properties",
+        show_error: bool = False,
     ) -> dict:
         """Get a node based on the parent ID and name. This method does basically
            a query with "where_name" and the "result" is a list.
@@ -1714,6 +1716,7 @@ class OTCS(object):
         Args:
             parent_id (integer) is the node Id of the parent node
             name (string) is the name of the node to get
+            fields (string): which fields to retrieve. This can have a big impact on performance!
             show_error (boolean, optional): treat as error if node is not found
         Returns:
             dictionary: Node information or None if no node with this name is found in parent.
@@ -1722,6 +1725,8 @@ class OTCS(object):
 
         # Add query parameters (these are NOT passed via JSon body!)
         query = {"where_name": name}
+        if fields:
+            query["fields"] = fields
         encoded_query = urllib.parse.urlencode(query, doseq=True)
 
         request_url = (
@@ -1809,7 +1814,7 @@ class OTCS(object):
 
     # end method definition
 
-    def getNodeByVolumeAndPath(self, volume_type: int, path: list) -> dict:
+    def getNodeByVolumeAndPath(self, volume_type: int, path: list = []) -> dict:
         """Get a node based on the volume and path (list of container items).
 
         Args:
@@ -1830,6 +1835,7 @@ class OTCS(object):
                 "Physical Objects Workspace"        = 413
                 "Extended ECM"                      = 882
                 "Enterprise Workspace"              = 141
+                "Personal Workspace"                = 142
                 "Business Workspaces"               = 862
             path (list): list of container items (top down), last item is name of to be retrieved item.
                          If path is empty the node of the volume is returned.
@@ -1918,6 +1924,7 @@ class OTCS(object):
         show_hidden: bool = False,
         limit: int = 100,
         page: int = 1,
+        fields: str = "properties",  # per default we just get the most important information
     ) -> dict:
         """Get a subnodes of a parent node ID.
 
@@ -1931,6 +1938,7 @@ class OTCS(object):
             show_hidden (boolean, optional): list also hidden items (default = False)
             limit (integer, optional): maximum number of results (default = 100)
             page (integer, optional): number of result page (default = 1 = 1st page)
+            fields (string): which fields to retrieve. This can have a big impact on performance!
         Returns:
             dictionary: Subnodes information or None if no node with this parent ID is found.
         """
@@ -1946,6 +1954,8 @@ class OTCS(object):
             query["show_hidden"] = show_hidden
         if page > 1:
             query["page"] = page
+        if fields:
+            query["fields"] = fields
 
         encodedQuery = urllib.parse.urlencode(query, doseq=True)
 
@@ -2574,7 +2584,11 @@ class OTCS(object):
         if not version_number:
             response = self.getLatestDocumentVersion(node_id)
             if not response:
-                logger.error("Cannot get latest version of document with ID -> {}".format(node_id))
+                logger.error(
+                    "Cannot get latest version of document with ID -> {}".format(
+                        node_id
+                    )
+                )
             version_number = response["data"]["version_number"]
 
         request_url = (
@@ -2583,8 +2597,7 @@ class OTCS(object):
             + str(node_id)
             + "/versions/"
             + str(version_number)
-            + "/content/"
-            + str(node_id)
+            + "/content"
         )
         request_header = self.requestDownloadHeader()
 
@@ -3194,7 +3207,7 @@ class OTCS(object):
         logger.info("Deploy workbench -> {} ({})".format(workbench_name, workbench_id))
         response = self.deployWorkbench(workbench_id)
         if response == None:
-            logger.warning("Failed to to deploy workbench -> {}".format(workbench_name))
+            logger.error("Failed to deploy workbench -> {}".format(workbench_name))
             return None
 
         logger.info(
@@ -3212,52 +3225,6 @@ class OTCS(object):
 
     # end method definition
 
-    def replaceInXmlFiles(
-        self, directory: str, search_pattern: str, replace_string: str
-    ) -> bool:
-        """Replaces all occurrences of the search pattern with the replace string in all XML files
-            in the directory and its subdirectories.
-
-        Args:
-            directory (string): directory to traverse for XML files
-            search_pattern (sting): string to search in the XML file
-            replace_string (string): replacement string
-        Returns:
-            boolean: True if a replacement happened, False otherwise
-        """
-        # Define the regular expression pattern to search for
-        pattern = re.compile(search_pattern)
-        found = False
-
-        # Traverse the directory and its subdirectories
-        for subdir, dirs, files in os.walk(directory):
-            for file in files:
-                # Check if the file is an XML file
-                if file.endswith(".xml"):
-                    # Read the contents of the file
-                    file_path = os.path.join(subdir, file)
-                    with open(file_path, "r") as f:
-                        contents = f.read()
-
-                    # Replace all occurrences of the search pattern with the replace string
-                    new_contents = pattern.sub(replace_string, contents)
-
-                    # Write the updated contents to the file if there were replacements
-                    if contents != new_contents:
-                        logger.info(
-                            "Found search string -> {} in XML file -> {}. Updating content...".format(
-                                search_pattern, file_path
-                            )
-                        )
-                        # Write the updated contents to the file
-                        with open(file_path, "w") as f:
-                            f.write(new_contents)
-                        found = True
-
-        return found
-
-        # end method definition
-
     def replaceTransportPlaceholders(
         self, zip_file_path: str, replacements: list
     ) -> bool:
@@ -3285,38 +3252,77 @@ class OTCS(object):
 
         # Replace search pattern with replace string in all XML files in the directory and its subdirectories
         for replacement in replacements:
-            if replacement["placeholder"] == replacement["value"]:
+            if not "value" in replacement:
+                logger.error("Replacement needs a value but it is not specified. Skipping...")
+                continue
+            if "enabled" in replacement and not replacement["enabled"]:
                 logger.info(
-                    "Placeholder and replacement are identical -> {}. Skipping...".format(
-                        replacement["value"]
-                    )
+                    "Replacement for transport -> {} is disabled. Skipping...".format(zip_file_path)
                 )
                 continue
-            logger.info(
-                "Replace -> {} with -> {} in Transport package -> {}".format(
-                    replacement["placeholder"], replacement["value"], zip_file_folder
+            # there are two types of replacements:
+            # 1. XPath - more elegant and powerful
+            # 2. Search & Replace - basically treat the XML file like a like file and do a search & replace
+            if "xpath" in replacement:
+                logger.info(
+                    "Using xpath -> {} to narrow down the replacement".format(
+                        replacement["xpath"]
+                    )
                 )
-            )
-            found = self.replaceInXmlFiles(
-                zip_file_folder, replacement["placeholder"], replacement["value"]
+                if "setting" in replacement:
+                    logger.info(
+                        "Looking up setting -> {} in XML element".format(
+                            replacement["setting"]
+                        )
+                    )
+                if "assoc_elem" in replacement:
+                    logger.info(
+                        "Looking up assoc element -> {} in XML element".format(
+                            replacement["assoc_elem"]
+                        )
+                    )
+            else: # we have a simple "search & replace" replacement
+                if not "placeholder" in replacement:
+                    logger.error("Replacement without an xpath needs a placeholder value but it is not specified. Skipping...")
+                    continue
+                if replacement.get("placeholder") == replacement["value"]:
+                    logger.info(
+                        "Placeholder and replacement are identical -> {}. Skipping...".format(
+                            replacement["value"]
+                        )
+                    )
+                    continue
+                logger.info(
+                    "Replace -> {} with -> {} in Transport package -> {}".format(
+                        replacement["placeholder"], replacement["value"], zip_file_folder
+                    )
+                )
+
+            found = XML.replaceInXmlFiles(
+                zip_file_folder,
+                replacement.get("placeholder"),
+                replacement["value"],
+                replacement.get("xpath"),
+                replacement.get("setting"),
+                replacement.get("assoc_elem"),
             )
             if found:
                 logger.info(
-                    "Found replacement string -> {} in Transport package -> {}".format(
-                        replacement["placeholder"], zip_file_folder
+                    "Replacement -> {} has been completed successfully for Transport package -> {}".format(
+                        replacement, zip_file_folder
                     )
                 )
                 modified = True
             else:
                 logger.warning(
-                    "Did not find replacement string -> {} in Transport package -> {}".format(
-                        replacement["placeholder"], zip_file_folder
+                    "Replacement -> {} failed for Transport package -> {}".format(
+                        replacement, zip_file_folder
                     )
                 )
 
         if not modified:
             logger.warning(
-                "None of the replacements have been found in transport -> {}".format(
+                "None of the specified replacements have been successful for Transport package -> {}. No need to create a new transport package.".format(
                     zip_file_folder
                 )
             )
@@ -4159,13 +4165,13 @@ class OTCS(object):
         if not os.path.exists(file_path):
             logger.error("Workdpace icon file does not exist -> {}".format(file_path))
             return None
-        
-#        icon_file = open(file_path, "rb")
+
+        #        icon_file = open(file_path, "rb")
 
         updateWorkspaceIconPutBody = {
             "file_content_type": file_mimetype,
             "file_filename": os.path.basename(file_path),
-            "file": file_path #icon_file
+            "file": file_path,  # icon_file
         }
 
         request_url = (