pyxecm 1.4__py3-none-any.whl → 1.5__py3-none-any.whl

pyxecm/helper/web.py

@@ -20,6 +20,7 @@ import logging
 import socket
 import time
 import requests
+from lxml import html
 
 logger = logging.getLogger("pyxecm.web")
 
@@ -47,7 +48,7 @@ class HTTP(object):
             bool: True is reachable, False otherwise
         """
 
-        logger.info(
+        logger.debug(
             "Test if host -> %s is reachable on port -> %s ...", hostname, str(port)
         )
         try:
@@ -67,7 +68,7 @@ class HTTP(object):
             )
             return False
         else:
-            logger.info("Host is reachable at -> %s:%s", hostname, str(port))
+            logger.debug("Host is reachable at -> %s:%s", hostname, str(port))
             return True
 
     # end method definition
@@ -81,6 +82,8 @@ class HTTP(object):
         timeout: int = 60,
         retries: int = 0,
         wait_time: int = 0,
+        wait_on_status: list | None = None,
+        show_error: bool = True,
     ):
         """Issues an http request to a given URL.
 
@@ -93,6 +96,9 @@ class HTTP(object):
             timeout (int, optional): timeout in seconds
             retries (int, optional): number of retries. If -1 then unlimited retries.
             wait_time (int, optional): number of seconds to wait after each try
+            wait_on_status (list, optional): list of status codes we want to wait on. If None
+                                             or empty then we wait for all return codes if
+                                             wait_time > 0
         Returns:
             Response of call
         """
@@ -100,61 +106,179 @@ class HTTP(object):
         if not headers:
             headers = requestHeaders
 
-        logger.info(
-            "Make HTTP Request to URL -> %s using -> %s method with payload -> %s (max number of retries = %s)",
-            url,
-            method,
-            str(payload),
-            str(retries),
+        message = "Make HTTP Request to URL -> {} using -> {} method".format(
+            url, method
         )
+        if payload:
+            message += " with payload -> {}".format(payload)
+        if retries:
+            message += " (max number of retries -> {}, wait time between retries -> {})".format(
+                retries, wait_time
+            )
+            try:
+                retries = int(retries)
+            except ValueError:
+                logger.warning(
+                    "HTTP request -> retries is not a valid integer value: %s, defaulting to 0 retries ",
+                    retries,
+                )
+                retries = 0
+
+        logger.debug(message)
 
         try_counter = 1
 
         while True:
-            response = requests.request(
-                method=method, url=url, data=payload, headers=headers, timeout=timeout
-            )
-
-            if not response.ok and retries == 0:
-                logger.error(
-                    "HTTP request -> %s to url -> %s failed; status -> %s; error -> %s",
-                    method,
-                    url,
-                    response.status_code,
-                    response.text,
+            try:
+                response = requests.request(
+                    method=method,
+                    url=url,
+                    data=payload,
+                    headers=headers,
+                    timeout=timeout,
                 )
-                return response
-
-            elif response.ok:
-                logger.info(
-                    "HTTP request -> %s to url -> %s succeeded with status -> %s!",
-                    method,
-                    url,
-                    response.status_code,
-                )
-                if wait_time > 0:
-                    logger.info("Sleeping %s seconds...", wait_time)
-                    time.sleep(wait_time)
-                return response
-
-            else:
+                logger.debug("%s", response.text)
+            except Exception as exc:
+                response = None
                 logger.warning(
-                    "HTTP request -> %s to url -> %s failed (try %s); status -> %s; error -> %s",
+                    "HTTP request -> %s to url -> %s failed failed (try %s); error -> %s",
                     method,
                     url,
                     try_counter,
-                    response.status_code,
-                    response.text,
+                    exc,
                 )
-                if wait_time > 0:
-                    logger.warning(
-                        "Sleeping %s seconds and then trying once more...",
-                        str(wait_time),
+
+                # do we have an error and don't want to retry?
+            if response is not None:
+                # Do we have a successful result?
+                if response.ok:
+                    logger.debug(
+                        "HTTP request -> %s to url -> %s succeeded with status -> %s!",
+                        method,
+                        url,
+                        response.status_code,
                     )
-                    time.sleep(wait_time)
-                else:
-                    logger.warning("Trying once more...")
-                retries -= 1
-                try_counter += 1
+
+                    if wait_on_status and response.status_code in wait_on_status:
+                        logger.debug(
+                            "%s is in wait_on_status list: %s",
+                            response.status_code,
+                            wait_on_status,
+                        )
+                    else:
+                        return response
+
+                elif not response.ok:
+                    message = "HTTP request -> {} to url -> {} failed; status -> {}; error -> {}".format(
+                        method,
+                        url,
+                        response.status_code,
+                        (
+                            response.text
+                            if response.headers.get("content-type")
+                            == "application/json"
+                            else "see debug log"
+                        ),
+                    )
+                    if show_error and retries == 0:
+                        logger.error(message)
+                    else:
+                        logger.warning(message)
+
+            # Check if another retry is allowed, if not return None
+            if retries == 0:
+                return None
+
+            if wait_time > 0:
+                logger.warning(
+                    "Sleeping %s seconds and then trying once more...",
+                    str(wait_time),
+                )
+                time.sleep(wait_time)
+
+            retries -= 1
+            try_counter += 1
+
+    # end method definition
+
+    def download_file(
+        self,
+        url: str,
+        filename: str,
+        timeout: int = 120,
+        retries: int = 0,
+        wait_time: int = 0,
+        wait_on_status: list | None = None,
+        show_error: bool = True,
+    ) -> bool:
+        """Download a file from a URL
+
+        Args:
+            url (str): URL
+            filename (str): filename to save
+            timeout (int, optional): timeout in seconds
+            retries (int, optional): number of retries. If -1 then unlimited retries.
+            wait_time (int, optional): number of seconds to wait after each try
+            wait_on_status (list, optional): list of status codes we want to wait on. If None
+                                             or empty then we wait for all return codes if
+                                             wait_time > 0
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+
+        response = self.http_request(
+            url=url,
+            method="GET",
+            retries=retries,
+            timeout=timeout,
+            wait_time=wait_time,
+            wait_on_status=wait_on_status,
+            show_error=show_error,
+        )
+
+        if response is None:
+            return False
+
+        if response.ok:
+            with open(filename, "wb") as f:
+                f.write(response.content)
+            logger.debug("File downloaded successfully as -> %s", filename)
+            return True
+
+        return False
 
     # end method definition
+
+    def extract_content(self, url: str, xpath: str) -> str | None:
+        """Extract a string from a response of a HTTP request
+           based on an XPath.
+
+        Args:
+            url (str): URL to open
+            xpath (str): XPath expression to apply to the result
+
+        Returns:
+            str | None: Extracted string or None in case of an error.
+        """
+
+        # Send a GET request to the URL
+        response = requests.get(url, timeout=None)
+
+        # Check if request was successful
+        if response.status_code == 200:
+            # Parse the HTML content
+            tree = html.fromstring(response.content)
+
+            # Extract content using XPath
+            elements = tree.xpath(xpath)
+
+            # Get text content of all elements and join them
+            content = "\n".join([elem.text_content().strip() for elem in elements])
+
+            # Return the extracted content
+            return content
+        else:
+            # If request was not successful, print error message
+            logger.error(response.status_code)
+            return None

pyxecm/helper/xml.py

@@ -3,6 +3,10 @@
 Class: XML
 Methods:
 
+load_xml_file: Load an XML file into a Python list of dictionaries
+load_xml_files_from_directory: Load all XML files from a directory that matches defined file names
+                               then using the XPath to identify a set of elements and convert them
+                               into a Python list of dictionaries.
 get_xml_element: Retrieve an XML Element from a string using an XPath expression
 modify_xml_element: Update the text (= content) of an XML element
 search_setting: Search a JSON-like setting inside an XML text telement
@@ -21,10 +25,12 @@ __email__ = "mdiefenb@opentext.com"
 import logging
 import os
 import re
+import fnmatch
 
 # we need lxml instead of stadard xml.etree to have xpath capabilities!
 from lxml import etree
 import xmltodict
+import zipfile
 
 # import xml.etree.ElementTree as etree
 from pyxecm.helper.assoc import Assoc
@@ -33,7 +39,129 @@ logger = logging.getLogger("pyxecm.xml")
 
 
 class XML:
-    """XML Class to parse and update Extended ECM transport packages"""
+    """XML Class to handle XML processing, e.g. to parse and update Extended ECM transport packages"""
+
+    @classmethod
+    def load_xml_file(
+        cls, file_path: str, xpath: str, dir_name: str | None = None
+    ) -> list | None:
+        """Load an XML file into a Python list of dictionaries
+
+        Args:
+            file_path (str): Path to XML file
+            xpath (str): XPath to select sub-elements
+
+        Returns:
+            dict | None: _description_
+        """
+
+        try:
+
+            tree = etree.parse(file_path)
+            if not tree:
+                return []
+
+            # Perform the XPath query to select 'child' elements
+            elements = tree.xpath(xpath)  # Adjust XPath as needed
+
+            # Convert the selected elements to dictionaries
+            results = []
+            tag = xpath.split("/")[-1]
+            for element in elements:
+                element_dict = xmltodict.parse(etree.tostring(element))
+                if tag in element_dict:
+                    element_dict = element_dict[tag]
+                if dir_name:
+                    element_dict["directory"] = dir_name
+                results.append(element_dict)
+
+        except IOError as e:
+            logger.error("IO Error -> %s", str(e))
+        except etree.XMLSyntaxError as e:
+            logger.error("XML Syntax Error -> %s", str(e))
+        except etree.DocumentInvalid as e:
+            logger.error("Document Invalid -> %s", str(e))
+
+        return results
+
+    # end method definition
+
+    @classmethod
+    def load_xml_files_from_directory(
+        cls, path_to_root: str, filenames: list | None, xpath: str | None = None
+    ) -> list | None:
+        """Load all XML files from a directory that matches defined file names
+           then using the XPath to identify a set of elements and convert them
+           into a Python list of dictionaries.
+
+        Args:
+            path_to_root (str): Path to the root element of the
+                                directory structure
+            filenames (list): list of filenames. If empty all filenames ending
+                              with ".xml" are used.
+            xpath (str, optional): XPath to the elements we want to select
+
+        Returns:
+            list: List of dictionaries
+        """
+
+        try:
+
+            # Check if the provided path is a directory
+            if not os.path.isdir(path_to_root) and not path_to_root.endswith(".zip"):
+                logger.error(
+                    "The provided path '%s' is not a valid directory or Zip file.",
+                    path_to_root,
+                )
+                return False
+
+            if path_to_root.endswith(".zip"):
+                zip_file_folder = os.path.splitext(path_to_root)[0]
+                if not os.path.exists(zip_file_folder):
+                    logger.info(
+                        "Unzipping -> '%s' into folder -> '%s'...",
+                        path_to_root,
+                        zip_file_folder,
+                    )
+                    with zipfile.ZipFile(path_to_root, "r") as zfile:
+                        zfile.extractall(zip_file_folder)
+                else:
+                    logger.info(
+                        "Zip file is already extracted (path -> '%s' exists). Reusing extracted data...",
+                        zip_file_folder,
+                    )
+                path_to_root = zip_file_folder
+
+            results = []
+
+            # Walk through the directory
+            for root, _, files in os.walk(path_to_root):
+                for file_data in files:
+                    file_path = os.path.join(root, file_data)
+                    file_size = os.path.getsize(file_path)
+                    file_name = os.path.basename(file_path)
+                    dir_name = os.path.dirname(file_path)
+
+                    if any(
+                        fnmatch.fnmatch(file_path, pattern) for pattern in filenames
+                    ) and file_name.endswith(".xml"):
+                        logger.info(
+                            "Load XML file -> '%s' of size -> %s", file_path, file_size
+                        )
+                        results += cls.load_xml_file(
+                            file_path, xpath=xpath, dir_name=dir_name
+                        )
+
+        except NotADirectoryError as nde:
+            logger.error("Error -> %s", str(nde))
+        except FileNotFoundError as fnfe:
+            logger.error("Error -> %s", str(fnfe))
+        except PermissionError as pe:
+            logger.error("Error -> %s", str(pe))
+
+        return results
+
+    # end method definition
 
     @classmethod
     def get_xml_element(cls, xml_content: str, xpath: str):
@@ -55,6 +183,8 @@ class XML:
 
         return element
 
+    # end method definition
+
     @classmethod
     def modify_xml_element(cls, xml_content: str, xpath: str, new_value: str):
         """Update the text (= content) of an XML element
@@ -72,6 +202,8 @@ class XML:
         else:
             logger.warning("XML Element -> %s not found.", xpath)
 
+    # end method definition
+
     @classmethod
     def search_setting(
         cls,
@@ -122,6 +254,8 @@ class XML:
         else:
             return None
 
+    # end method definition
+
     @classmethod
     def replace_setting(
         cls,
@@ -170,6 +304,8 @@ class XML:
 
         return new_text
 
+    # end method definition
+
     @classmethod
     def replace_in_xml_files(
         cls,
@@ -216,8 +352,8 @@ class XML:
                     # if xpath is given we do an intelligent replacement
                     if xpath:
                         xml_modified = False
-                        logger.info("Replacement with xpath...")
-                        logger.info(
+                        logger.debug("Replacement with xpath...")
+                        logger.debug(
                             "XML path -> %s, setting -> %s, assoc element -> %s",
                             xpath,
                             setting,
@@ -225,17 +361,15 @@ class XML:
                         )
                         tree = etree.parse(file_path)
                         if not tree:
-                            logger.erro(
-                                "Cannot parse XML tree -> {}. Skipping...".format(
-                                    file_path
-                                )
+                            logger.error(
+                                "Cannot parse XML tree -> %s. Skipping...", file_path
                             )
                             continue
                         root = tree.getroot()
                         # find the matching XML elements using the given XPath:
                         elements = root.xpath(xpath)
                         if not elements:
-                            logger.info(
+                            logger.debug(
                                 "The XML file -> %s does not have any element with the given XML path -> %s. Skipping...",
                                 file_path,
                                 xpath,
@@ -243,7 +377,7 @@ class XML:
                             continue
                         for element in elements:
                             # as XPath returns a list
-                            logger.info(
+                            logger.debug(
                                 "Found XML element -> %s in file -> %s using xpath -> %s",
                                 element.tag,
                                 filename,
@@ -251,7 +385,7 @@ class XML:
                             )
                             # the simple case: replace the complete text of the XML element
                             if not setting and not assoc_elem:
-                                logger.info(
+                                logger.debug(
                                     "Replace complete text of XML element -> %s from -> %s to -> %s",
                                     xpath,
                                     element.text,
@@ -261,7 +395,7 @@ class XML:
                                 xml_modified = True
                             # In this case we want to set a complete value of a setting (basically replacing a whole line)
                             elif setting and not assoc_elem:
-                                logger.info(
+                                logger.debug(
                                     "Replace single setting -> %s in XML element -> %s with new value -> %s",
                                     setting,
                                     xpath,
@@ -271,7 +405,7 @@ class XML:
                                     element.text, setting, is_simple=True
                                 )
                                 if setting_value:
-                                    logger.info(
+                                    logger.debug(
                                         "Found existing setting value -> %s",
                                         setting_value,
                                     )
@@ -290,7 +424,7 @@ class XML:
                                         replace_setting = (
                                             '"' + setting + '":"' + replace_string + '"'
                                         )
-                                    logger.info(
+                                    logger.debug(
                                         "Replacement setting -> %s", replace_setting
                                     )
                                     element.text = cls.replace_setting(
@@ -308,7 +442,7 @@ class XML:
                                     continue
                             # in this case the text is just one assoc (no setting substructure)
                             elif not setting and assoc_elem:
-                                logger.info(
+                                logger.debug(
                                     "Replace single Assoc value -> %s in XML element -> %s with -> %s",
                                     assoc_elem,
                                     xpath,
@@ -322,13 +456,13 @@ class XML:
                                     assoc_string=assoc_string
                                 )
                                 logger.debug("Assoc Dict -> %s", str(assoc_dict))
-                                assoc_dict[
-                                    assoc_elem
-                                ] = replace_string  # escaped_replace_string
+                                assoc_dict[assoc_elem] = (
+                                    replace_string  # escaped_replace_string
+                                )
                                 assoc_string_new: str = Assoc.dict_to_string(
                                     assoc_dict=assoc_dict
                                 )
-                                logger.info(
+                                logger.debug(
                                     "Replace assoc with -> %s", assoc_string_new
                                 )
                                 element.text = assoc_string_new
@@ -336,7 +470,7 @@ class XML:
                                 xml_modified = True
                             # In this case we have multiple settings with their own assocs
                             elif setting and assoc_elem:
-                                logger.info(
+                                logger.debug(
                                     "Replace single Assoc value -> %s in setting -> %s in XML element -> %s with -> %s",
                                     assoc_elem,
                                     setting,
@@ -347,7 +481,7 @@ class XML:
                                     element.text, setting, is_simple=False
                                 )
                                 if setting_value:
-                                    logger.info(
+                                    logger.debug(
                                         "Found setting value -> %s", setting_value
                                     )
                                     assoc_string: str = Assoc.extract_assoc_string(
@@ -361,13 +495,13 @@ class XML:
                                     escaped_replace_string = replace_string.replace(
                                         "'", "\\\\\u0027"
                                     )
-                                    logger.info(
+                                    logger.debug(
                                         "Escaped replacement string -> %s",
                                         escaped_replace_string,
                                     )
-                                    assoc_dict[
-                                        assoc_elem
-                                    ] = escaped_replace_string  # escaped_replace_string
+                                    assoc_dict[assoc_elem] = (
+                                        escaped_replace_string  # escaped_replace_string
+                                    )
                                     assoc_string_new: str = Assoc.dict_to_string(
                                         assoc_dict=assoc_dict
                                     )
@@ -378,7 +512,7 @@ class XML:
                                     replace_setting = (
                                         '"' + setting + '":"' + assoc_string_new + '"'
                                     )
-                                    logger.info(
+                                    logger.debug(
                                         "Replacement setting -> %s", replace_setting
                                     )
                                     # here we need to apply a "trick". It is required
@@ -407,7 +541,7 @@ class XML:
                                     )
                                     continue
                         if xml_modified:
-                            logger.info(
+                            logger.debug(
                                 "XML tree has been modified. Write updated file -> %s...",
                                 file_path,
                             )
@@ -465,7 +599,7 @@ class XML:
                             found = True
                     # this is not using xpath - do a simple search and replace
                     else:
-                        logger.info("Replacement without xpath...")
+                        logger.debug("Replacement without xpath...")
                         with open(file_path, "r", encoding="UTF-8") as f:
                             contents = f.read()
                         # Replace all occurrences of the search pattern with the replace string
@@ -473,7 +607,7 @@ class XML:
 
                         # Write the updated contents to the file if there were replacements
                         if contents != new_contents:
-                            logger.info(
+                            logger.debug(
                                 "Found search string -> %s in XML file -> %s. Write updated file...",
                                 search_pattern,
                                 file_path,
@@ -485,6 +619,8 @@ class XML:
 
         return found
 
+    # end method definition
+
     @classmethod
     def extract_from_xml_files(
         cls,
@@ -511,18 +647,18 @@ class XML:
                     # Read the contents of the file
                     file_path = os.path.join(subdir, filename)
 
-                    logger.info("Extraction with xpath -> %s...", xpath)
+                    logger.debug("Extraction with xpath -> %s...", xpath)
                     tree = etree.parse(file_path)
                     if not tree:
-                        logger.erro(
-                            "Cannot parse XML tree -> {}. Skipping...".format(file_path)
+                        logger.error(
+                            "Cannot parse XML file -> '%s'. Skipping...", file_path
                         )
                         continue
                     root = tree.getroot()
                     # find the matching XML elements using the given XPath:
                     elements = root.xpath(xpath)
                     if not elements:
-                        logger.info(
+                        logger.debug(
                             "The XML file -> %s does not have any element with the given XML path -> %s. Skipping...",
                             file_path,
                             xpath,
@@ -530,7 +666,7 @@ class XML:
                         continue
                     for element in elements:
                         # as XPath returns a list
-                        logger.info(
+                        logger.debug(
                             "Found XML element -> %s in file -> %s using xpath -> %s. Add it to result list.",
                             element.tag,
                             filename,
@@ -551,4 +687,4 @@ class XML:
 
         return extracted_data_list
 
-        # end method definition
+    # end method definition