pdmv-http-client 2.1.0__py3-none-any.whl

rest/applications/mcm/core.py

@@ -0,0 +1,277 @@
+"""
+REST client for the McM application.
+"""
+
+import warnings
+from typing import Union
+
+from rest.applications.base import BaseClient
+
+
+class McM(BaseClient):
+    """
+    Initializes an HTTP client for querying McM.
+    """
+
+    def __init__(
+        self,
+        id: str = BaseClient.SSO,
+        debug: bool = False,
+        cookie: Union[str, None] = None,
+        dev: bool = True,
+        client_id: str = "",
+        client_secret: str = "",
+    ):
+        # Set the HTTP session
+        super().__init__(
+            app="mcm",
+            id=id,
+            debug=debug,
+            cookie=cookie,
+            dev=dev,
+            client_id=client_id,
+            client_secret=client_secret,
+        )
+
+    def __get(self, url):
+        warnings.warn(
+            "This name mangled method will be removed in the future, use self._get(...) instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._get(url=url)
+
+    def __put(self, url, data):
+        warnings.warn(
+            "This name mangled method will be removed in the future, use self._put(...) instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._put(url=url, data=data)
+
+    def __post(self, url, data):
+        warnings.warn(
+            "This name mangled method will be removed in the future, use self._post(...) instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._post(url=url, data=data)
+
+    def __delete(self, url):
+        warnings.warn(
+            "This name mangled method will be removed in the future, use self._delete(...) instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._delete(url=url)
+
+    # McM methods
+    def get(self, object_type, object_id=None, query="", method="get", page=-1):
+        """
+        Get data from McM
+        object_type - [chained_campaigns, chained_requests, campaigns, requests, flows, etc.]
+        object_id - usually prep id of desired object
+        query - query to be run in order to receive an object, e.g. tags=M17p1A, multiple parameters can be used with & tags=M17p1A&pwg=HIG
+        method - action to be performed, such as get, migrate or inspect
+        page - which page to be fetched. -1 means no pagination, return all results
+        """
+        object_type = object_type.strip()
+        if object_id:
+            object_id = object_id.strip()
+            self.logger.debug(
+                "Object ID %s provided, method is %s, database %s",
+                object_id,
+                method,
+                object_type,
+            )
+            url = "restapi/%s/%s/%s" % (object_type, method, object_id)
+            result = self._get(url)
+            if type(result) == list:
+                return result
+            elif type(result) == str:
+                return result
+            result = result.get("results")
+            if not result:
+                return None
+            return result
+        elif query:
+            if page != -1:
+                self.logger.debug(
+                    "Fetching page %s of %s for query %s", page, object_type, query
+                )
+                url = "search/?db_name=%s&limit=50&page=%d&%s" % (
+                    object_type,
+                    page,
+                    query,
+                )
+                results = self._get(url).get("results", [])
+                self.logger.debug(
+                    "Found %s %s in page %s for query %s",
+                    len(results),
+                    object_type,
+                    page,
+                    query,
+                )
+                return results
+            else:
+                self.logger.debug(
+                    "Page not given, will use pagination to build response"
+                )
+                page_results = [{}]
+                results = []
+                page = 0
+                while page_results:
+                    page_results = self.get(
+                        object_type=object_type, query=query, method=method, page=page
+                    )
+                    results += page_results
+                    page += 1
+
+                return results
+        else:
+            self.logger.error("Neither object ID, nor query is given, doing nothing...")
+
+    def update(self, object_type, object_data):
+        """
+        Update data in McM
+        object_type - [chained_campaigns, chained_requests, campaigns, requests, flows, etc.]
+        object_data - new JSON of an object to be updated
+        """
+        return self.put(object_type, object_data, method="update")
+
+    def put(self, object_type, object_data, method="save"):
+        """
+        Put data into McM
+        object_type - [chained_campaigns, chained_requests, campaigns, requests, flows, etc.]
+        object_data - new JSON of an object to be updated
+        method - action to be performed, default is 'save'
+        """
+        url = "restapi/%s/%s" % (object_type, method)
+        res = self._put(url, object_data)
+        return res
+
+    def post(self, object_type, object_data, method):
+        """
+        Post data into McM
+        object_type - [chained_campaigns, chained_requests, campaigns, requests, flows, etc.]
+        object_data - JSON to be posted
+        method - restapi to be called
+        """
+        url = "restapi/%s/%s" % (object_type, method)
+        res = self._post(url, object_data)
+        return res
+
+    def approve(self, object_type, object_id, level=None):
+        if level is None:
+            url = "restapi/%s/approve/%s" % (object_type, object_id)
+        else:
+            url = "restapi/%s/approve/%s/%d" % (object_type, object_id, level)
+
+        return self._get(url)
+
+    def clone_request(self, object_data):
+        return self.put("requests", object_data, method="clone")
+
+    def get_range_of_requests(self, query):
+        res = self._put("restapi/requests/listwithfile", data={"contents": query})
+        return res.get("results", None)
+
+    def delete(self, object_type, object_id):
+        url = "restapi/%s/delete/%s" % (object_type, object_id)
+        self._delete(url)
+
+    def forceflow(self, prepid):
+        """
+        Force a flow on a chained request with given `prepid`
+        """
+        res = self._get("restapi/chained_requests/flow/%s/force" % (prepid))
+        return res.get("results", None)
+
+    def reset(self, prepid):
+        """
+        Reset a request
+        """
+        res = self._get("restapi/requests/reset/%s" % (prepid))
+        return res.get("results", None)
+
+    def soft_reset(self, prepid):
+        """
+        Soft reset a request
+        """
+        res = self._get("restapi/requests/soft_reset/%s" % (prepid))
+        return res.get("results", None)
+
+    def option_reset(self, prepid):
+        """
+        Option reset a request
+        """
+        res = self._get("restapi/requests/option_reset/%s" % (prepid))
+        return res.get("results", None)
+
+    def ticket_generate(self, ticket_prepid):
+        """
+        Generate chains for a ticket
+        """
+        res = self._get("restapi/mccms/generate/%s" % (ticket_prepid))
+        return res.get("results", None)
+
+    def ticket_generate_reserve(self, ticket_prepid):
+        """
+        Generate and reserve chains for a ticket
+        """
+        res = self._get("restapi/mccms/generate/%s/reserve" % (ticket_prepid))
+        return res.get("results", None)
+
+    def rewind(self, chained_request_prepid):
+        """
+        Rewind a chained request
+        """
+        res = self._get("restapi/chained_requests/rewind/%s" % (chained_request_prepid))
+        return res.get("results", None)
+
+    def flow(self, chained_request_prepid):
+        """
+        Flow a chained request
+        """
+        res = self._get("restapi/chained_requests/flow/%s" % (chained_request_prepid))
+        return res.get("results", None)
+
+    def root_requests_from_ticket(self, ticket_prepid):
+        """
+        Return list of all root (first ones in the chain) requests of a ticket
+        """
+        mccm = self.get("mccms", ticket_prepid)
+        query = ""
+        for root_request in mccm.get("requests", []):
+            if isinstance(root_request, str):
+                query += "%s\n" % (root_request)
+            elif isinstance(root_request, list):
+                # List always contains two elements - start and end of a range
+                query += "%s -> %s\n" % (root_request[0], root_request[1])
+            else:
+                self.logger.error(
+                    "%s is of unsupported type %s", root_request, type(root_request)
+                )
+
+        requests = self.get_range_of_requests(query)
+        return requests
+
+    def is_root_request(self, request: Union[str, dict]) -> bool:
+        """
+        Checks if a request is a root request.
+
+        Args:
+            request: Request prepid or request data.
+                The request must exists and its data must comply the expected
+                schema.
+        """
+        request_data: dict = {}
+        if isinstance(request, str):
+            # Received request prepid
+            request_data = self.get(object_type="requests", object_id=request) or {}
+        elif isinstance(request, dict):
+            request_data = request
+        else:
+            raise ValueError(f"Unexpected value: {request} - {type(request)}")
+
+        return request_data.get("type", "") in ("LHE", "Prod")

rest/applications/mcm/invalidate_request.py

@@ -0,0 +1,399 @@
+"""
+This module allows invalidating
+root requests and its generated chains.
+This is intended to be used only for
+`production_manager` users or `administrators`.
+"""
+
+import datetime
+import logging
+from copy import deepcopy
+from itertools import groupby
+from typing import Union
+
+from rest import McM
+from rest.utils.miscellaneous import pformat
+
+
+class InvalidateDeleteRequests:
+    """
+    Invalidate and delete all the chain requests in McM
+    linked to a root request in McM and delete the
+    root requests if required.
+    """
+
+    def __init__(self, mcm: McM) -> None:
+        self.mcm = mcm
+        self.logger = self._get_logger()
+        self._chain_request_type = "chained_requests"
+        self._request_type = "requests"
+        self.logger.warning(
+            "Sending requests to target environment: %s", self.mcm.server
+        )
+
+    def _get_logger(self) -> logging.Logger:
+        """
+        Creates a logger to record the performed steps.
+        """
+        logger: logging.Logger = logging.getLogger(__name__)
+        logger.handlers.clear()  # Avoid to record the same message twice in the log
+        formatter = logging.Formatter("[%(asctime)s][%(levelname)s] %(message)s")
+        current_date = str(datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
+        fh: logging.Handler = logging.FileHandler(
+            f"mcm_invalidate_requests_{current_date}.log"
+        )
+        fh.setFormatter(formatter)
+        logger.addHandler(fh)
+        return logger
+
+    def _get_invalidations(self, requests: list[str]) -> list[dict]:
+        """
+        Get all the invalidation records related to the given requests.
+
+        Args:
+            requests: List of `prepids` used to retrieve its invalidations.
+
+        Returns:
+            Invalidations related to the requests.
+        """
+        results: list[dict] = []
+        for req in requests:
+            inv_req = self.mcm.get("invalidations", query=f"prepid={req}")
+            assert isinstance(inv_req, list)
+            results += inv_req
+
+        return results
+
+    def _announce_invalidations(self, invalidations: list[dict]) -> dict:
+        """
+        Announce an invalidation for the given invalidation records.
+
+        Args:
+            invalidations: List of `invalidation` records to announce.
+
+        Returns:
+            Announce result.
+        """
+        # Flatten and just get the `_id` field
+        inv_ids: list[str] = [
+            i.get("_id", "") for i in invalidations if i.get("_id", "")
+        ]
+        announce_result = self.mcm.put(
+            object_type="invalidations", object_data=inv_ids, method="announce"
+        )
+
+        return announce_result
+
+    def _process_invalidation(self, request_prepids: list[str]) -> None:
+        """
+        Get all the invalidations related to a request and announces them
+        in case they exist.
+
+        Args:
+            request_prepids: List of request's prepid to process its
+                invalidation.
+        """
+        invalidations_to_announce = self._get_invalidations(requests=request_prepids)
+        if invalidations_to_announce:
+            announce_result = self._announce_invalidations(
+                invalidations=invalidations_to_announce
+            )
+            self.logger.info("Invalidation result: %s", announce_result)
+            if not announce_result.get("results"):
+                msg = f"Unable to invalidate records for all the following requests: {request_prepids}"
+                self.logger.error(msg)
+                self.logger.error("Invalidation records: %s", invalidations_to_announce)
+                raise RuntimeError(msg)
+        pass
+
+    def _invalidate_delete_chain_requests(
+        self, chain_req_data: list[dict], remove_chain: bool = False
+    ) -> None:
+        """
+        Rewinds the chain request to the root request
+        invalidating and deleting all the intermediate requests.
+
+        Also, deletes the chain request also if required.
+
+        Args:
+            chain_req_data: Chain requests data objects
+                to process.
+            remove_chain: Deletes the chain request after
+                perform the invalidation. If False, the chain request
+                disable `flag` is re-enabled.
+        """
+        chain_req_operate: list[dict] = []
+        chain_request_only_with_root: list[dict] = []
+        chain_request_more_than_root: list[dict] = []
+
+        # Filter the chain requests that only have the root request.
+        for chain_request in chain_req_data:
+            if len(chain_request.get("chain", [])) == 1:
+                chain_request_only_with_root.append(chain_request)
+            else:
+                chain_request_more_than_root.append(chain_request)
+
+        # Group the chained request by the second request in the chain
+        # (The first that is not the `root` request).
+        if chain_request_more_than_root:
+            grouped_chain = {
+                k: list(g)
+                for k, g in groupby(
+                    chain_request_more_than_root, lambda el: el["chain"][1]
+                )
+            }
+            if len(grouped_chain) != 1:
+                error_chained_request_prepids: list[str] = [
+                    ch["prepid"]
+                    for ch in chain_request_more_than_root
+                    if ch.get("prepid")
+                ]
+                msg = (
+                    "Unable to process chained requests. After grouping them by the second request, "
+                    f"more than one group has been found. This expects only one.\n"
+                    f"Request groups: {grouped_chain.keys()}\n"
+                    f"Chained requests: {pformat(error_chained_request_prepids)}"
+                )
+                self.logger.error(msg)
+                raise ValueError(msg)
+
+            # Include in the list to operate.
+            chain_req_operate += list(grouped_chain.values())[0]
+
+        if chain_request_only_with_root:
+            only_prepids = [el.get("prepid") for el in chain_request_only_with_root]
+            self.logger.warning(
+                "The following chain request only contain the root request in its chain: %s",
+                pformat(only_prepids),
+            )
+            chain_req_operate += chain_request_only_with_root
+
+        # Order the chain requests by `step` descending
+        chain_req_operate = sorted(
+            chain_req_operate, key=lambda el: el["step"], reverse=True
+        )
+        self.logger.info(
+            "Chain request pre-reset order (after sort): %s",
+            [el.get("prepid") for el in chain_req_operate],
+        )
+
+        # Set the flag to `False` and save
+        for ch_r in chain_req_operate:
+            ch_req_prepid = ch_r.get("prepid")
+            updated_ch_r = self.mcm.get(
+                object_type=self._chain_request_type, object_id=ch_req_prepid
+            )
+            updated_ch_r["action_parameters"]["flag"] = False
+            updated_ch_r = self.mcm.update(
+                object_type=self._chain_request_type, object_data=updated_ch_r
+            )
+            self.logger.info("Disable 'flag' response: %s", updated_ch_r)
+
+            if not updated_ch_r or not updated_ch_r.get("results"):
+                msg = f"Error updating chain requests: {ch_req_prepid}"
+                self.logger.error(msg)
+                raise RuntimeError(msg)
+
+        for ch_r in chain_req_operate:
+            ch_req_prepid = ch_r.get("prepid")
+
+            # Rewind the chain request to `root`.
+            rewind_endpoint = f"restapi/chained_requests/rewind_to_root/{ch_req_prepid}"
+            rewind_response = self.mcm._get(rewind_endpoint)
+            self.logger.info("Rewind chain request response: %s", rewind_response)
+            if not rewind_response or not rewind_response.get("results"):
+                msg = f"Unable to rewind chain request to root ({ch_r}) - Details: {rewind_response}"
+                self.logger.error(msg)
+                raise RuntimeError(msg)
+
+            # Announce the invalidation.
+            ch_req_requests: list[str] = ch_r.get("chain")
+            self._process_invalidation(request_prepids=ch_r.get("chain"))
+
+            # Delete the other requests EXCEPT for the `root`
+            # The first record in this list is the `root` request
+            chain_delete = ch_req_requests[1:]
+
+            # INFO: They must be deleted in order from the deepest
+            # data tier to upwards
+            chain_delete = reversed(chain_delete)
+
+            for rd in chain_delete:
+                self.mcm.delete(object_type=self._request_type, object_id=rd)
+
+        # Process the chain request
+        if remove_chain:
+            # Precondition: All the chained request share the same
+            # root request.
+            root_request_prepid: str = chain_req_operate[0].get("chain", [])[0]
+            self.logger.warning(
+                "Full resetting the root request (%s) as it is required to properly delete the chains",
+                root_request_prepid,
+            )
+            result = self.mcm.reset(prepid=root_request_prepid)
+            if not result:
+                raise RuntimeError("Unable to reset the root request", result)
+
+            # Process the root request invalidation.
+            self._process_invalidation([root_request_prepid])
+
+            # Remove all the chain requests
+            for ch_r in chain_req_operate:
+                ch_req_prepid = ch_r.get("prepid")
+                self.mcm.delete(
+                    object_type=self._chain_request_type, object_id=ch_req_prepid
+                )
+        else:
+            # Re-enable the `flag`
+            for ch_r in chain_req_operate:
+                ch_req_prepid = ch_r.get("prepid")
+                updated_ch_r = self.mcm.get(
+                    object_type=self._chain_request_type, object_id=ch_req_prepid
+                )
+                updated_ch_r["action_parameters"]["flag"] = True
+                updated_ch_r = self.mcm.update(
+                    object_type=self._chain_request_type, object_data=updated_ch_r
+                )
+                self.logger.info("Disable 'flag' response: %s", updated_ch_r)
+                if not updated_ch_r or not updated_ch_r.get("results"):
+                    msg = f"Error updating chain requests: {ch_req_prepid}"
+                    self.logger.error(msg)
+                    raise RuntimeError(msg)
+
+    def _invalidate_delete_root_request(
+        self, root_prepid: str, remove_root: bool = False, remove_chain: bool = False
+    ) -> None:
+        """
+        Invalidates and deletes all the chained request
+        linked to a root request and remove the root
+        request if required.
+
+        Args:
+            root_prepid: Root request prepid
+            remove_root: Invalidate and delete the root
+                request after processing its chains. If True,
+                the parameter `remove_chains` will be forced to
+                `True`
+            remove_chain: Delete the chain request after
+                processing it. If False, the chained requests
+                `flag` is re-enabled.
+        """
+        self.logger.info(
+            "Processing root request: %s, remove root: %s, remove chains: %s",
+            root_prepid,
+            remove_root,
+            remove_chain,
+        )
+
+        if remove_root and not remove_chain:
+            self.logger.warning(
+                "Setting `remove_chain` to True as a cascade delete is requested"
+            )
+            remove_chain = True
+
+        if remove_chain:
+            self.logger.warning(
+                (
+                    "Root request will be FULL RESET "
+                    "and its output dataset will be invalidated! "
+                    "This is required to delete all its related chained requests."
+                )
+            )
+
+        # 1. Get all the chained requests and process them.
+        chain_req: list[dict] = self.mcm.get(
+            object_type=self._chain_request_type, query=f"contains={root_prepid}"
+        )
+        self.logger.info(
+            "Request (%s), operating chain requests: %s",
+            root_prepid,
+            pformat([ch.get("prepid") for ch in chain_req]),
+        )
+        self._invalidate_delete_chain_requests(
+            chain_req_data=chain_req, remove_chain=remove_chain
+        )
+
+        # 2. Remove the root request if required.
+        if remove_root:
+            self.mcm.delete(object_type=self._request_type, object_id=root_prepid)
+
+    def _filter_root_requests(self, requests_prepid: list[str]) -> list[str]:
+        """
+        Filter the given request's prepids and only pick
+        those related only to root requests.
+        """
+        root_requests: list[str] = []
+        for rid in requests_prepid:
+            if self.mcm.is_root_request(request=rid):
+                root_requests.append(rid)
+        return root_requests
+
+    def invalidate_delete_cascade_requests(
+        self,
+        requests_prepid: list[str],
+        remove_root: bool = False,
+        remove_chain: bool = False,
+        limit: int = 2**64,
+    ) -> dict[str, list[str]]:
+        """
+        Invalidate and delete all the request including into the
+        chain requests related to the provided root request.
+        This could be considered as a "delete in cascade" process.
+
+        Args:
+            requests_prepid: List of root requests to process.
+                In case a non-root request is provided, it will be
+                filtered.
+            remove_root: After processing the chained requests,
+                invalidate the dataset related to the root request and
+                delete it.
+            remove_chain: After removing the intermediate requests,
+                remove the chain or re-enable it chain again.
+            limit: Process root requests until the limit is reached.
+
+        Returns:
+            Details about the requests processed in three categories: Success, Failed, Filtered.
+        """
+        root_request_prepids: list[str] = self._filter_root_requests(
+            requests_prepid=deepcopy(requests_prepid)
+        )
+        discarded_prepids = list(set(requests_prepid) - set(root_request_prepids))
+        if discarded_prepids:
+            self.logger.info(
+                "Following requests are not valid root requests (%s): %s",
+                len(discarded_prepids),
+                pformat(discarded_prepids),
+            )
+
+        failed_requests: list[str] = []
+        success_requests: list[str] = []
+        total_to_process = len(root_request_prepids)
+        for idx, root_prepid in enumerate(root_request_prepids, start=1):
+            try:
+                self.logger.info(
+                    "(%s/%s) Processing root request", idx, total_to_process
+                )
+                self._invalidate_delete_root_request(
+                    root_prepid=root_prepid,
+                    remove_root=remove_root,
+                    remove_chain=remove_chain,
+                )
+                success_requests.append(root_prepid)
+            except Exception as e:
+                self.logger.error(
+                    "Unable to process root request (%s): %s",
+                    root_prepid,
+                    e,
+                    exc_info=True,
+                )
+                failed_requests.append(root_prepid)
+            finally:
+                if limit <= idx:
+                    self.logger.info("Early stopping processing root requests...")
+                    break
+
+        return {
+            "success": success_requests,
+            "failed": failed_requests,
+            "filtered": discarded_prepids,
+        }