flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/HTTPClient.py

@@ -0,0 +1,268 @@
+"""Basic HTTP Connection Request.
+
+        Example:
+
+        ```yaml
+        HTTPClient:
+          Group: ICIMSDownload
+          url: https://api.icims.com/customers/{customer_id}/forms/{form_data_id}.txt
+          customer_id: '5674'
+          form_data_id: '1731'
+          form_id: '1823'
+          associate_id: '518491'
+          auth_type: basic
+          use_proxy: false
+          use_async: true
+          credentials:
+            username: ICIMS_API_USERNAME
+            password: ICIMS_API_PASSWORD
+          as_dataframe: false
+          download: true
+          timeout: 360
+          destination:
+            directory: /home/ubuntu/symbits/icims/files/forms/{associate_id}/
+            filename: '{form_id}_{filename}.html'
+            overwrite: false
+          no_errors:
+            '403':
+              errorMessage: This form has been disabled
+              errorCode: 3
+        ```
+
+    """
+import asyncio
+# configuration and settings
+from pathlib import PurePath
+from typing import Union
+from collections.abc import Callable
+from urllib.parse import urlencode
+from navconfig import config
+from querysource.types import strtobool
+from ..exceptions import DataNotFound, ComponentError, FileNotFound
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.http import HTTPService
+
+
+class HTTPClient(DownloadFromBase, HTTPService):
+    """
+    HTTPClient.
+
+    Overview
+
+           UserComponent: (abstract) ->
+
+    .. table:: Properties
+       :widths: auto
+
+    +-----------+----------+-----------+----------------------------------------------+
+    | Name      | Required | Summary                                                  |
+    +-----------+----------+-----------+----------------------------------------------+
+    | start     |   Yes    | It is executed when the component is "initialized",      |
+    |           |          | it MUST return TRUE if it does not fail                  |
+    +-----------+----------+-----------+----------------------------------------------+
+    | run       |   Yes    | Is the code that will execute the component ,must return TRUE or  |
+    |           |          | a content if it does not fail, but fails is declared      |
+    +-----------+----------+-----------+-----------------------------------------------+
+    | close     |   Yes    | It is used for any cleaning operation                     |
+    +-----------+----------+-----------+-----------------------------------------------+
+
+
+    Return the list of arbitrary days
+
+    """
+    accept: str = "application/xhtml+xml"
+    port: int = 80
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        # Credentials
+        self._user: str = None
+        self._pwd: str = None
+        # beautiful soup and html parser:
+        self._bs: Callable = None
+        self._parser: Callable = None
+        self._environment = config
+        # Return a Dataframe:
+        self.as_dataframe: bool = kwargs.pop('as_dataframe', True)
+        if isinstance(self.as_dataframe, str):
+            self.as_dataframe = strtobool(self.as_dataframe)
+        # Data:
+        self._data: dict = kwargs.pop("data", {})
+        # Credentials:
+        self.credentials: dict = kwargs.pop("credentials", {})
+        self.method: str = kwargs.pop("method", "get")
+        # calling parents
+        DownloadFromBase.__init__(
+            self,
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        HTTPService.__init__(
+            self,
+            **kwargs
+        )
+
+    async def evaluate_error(
+        self, response: Union[str, list], message: Union[str, list, dict]
+    ) -> tuple:
+        """evaluate_response.
+
+        Check Response status and available payloads.
+        Args:
+            response (_type_): _description_
+            url (str): _description_
+
+        Returns:
+            tuple: _description_
+        """
+        if isinstance(response, list):
+            # a list of potential errors:
+            for msg in response:
+                if message in msg:
+                    return True
+        if isinstance(response, dict) and "errors" in response:
+            errors = response["errors"]
+            if isinstance(errors, list):
+                for error in errors:
+                    try:
+                        if message in error:
+                            return True
+                    except TypeError:
+                        if message == error:
+                            return True
+            else:
+                if message == errors:
+                    return True
+        else:
+            if message in response:
+                return True
+        return False
+
+    def var_replacement(self):
+        """
+        Replaces variables in the arguments with their corresponding values.
+        """
+        for key, _ in self.arguments.items():
+            if key in self._variables:
+                self.arguments[key] = self._variables[key]
+
+    async def start(self, **kwargs):
+        """
+        Initializes the HTTPClient component, including fetching proxies if necessary.
+
+        :param kwargs: Additional keyword arguments.
+        :return: True if initialization is successful.
+        """
+        if self.use_proxy is True:
+            self._proxies = await self.get_proxies()
+        self.var_replacement()
+        await super(HTTPClient, self).start()
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        """
+        Executes the HTTPClient component, handling multiple URLs if provided.
+
+        :return: The result of the HTTP request(s).
+        """
+        if isinstance(self.url, list):
+            ## iterate over a list of URLs:
+            results = {}
+            for url in self.url:
+                uri = self.build_url(
+                    url, args=self.arguments, queryparams=urlencode(self.parameters)
+                )
+                try:
+                    if self.use_async is True:
+                        result, err = await self.async_request(uri, self.method, data=self._data)
+                    else:
+                        result, error = await self.request(uri, self.method, data=self._data)
+                    if not result:
+                        raise DataNotFound(
+                            f"Data was not found on: {uri}"
+                        )
+                    if error is not None:
+                        if isinstance(error, BaseException):
+                            raise error
+                        else:
+                            raise ComponentError(f"HTTPClient Error: {error}")
+                    ## processing results:
+                    if hasattr(self, "download"):
+                        ## add result to resultset
+                        results[result] = True
+                        if self._debug:
+                            self._logger.debug(f"File Exists > {result}")
+                    else:
+                        results[result] = result
+                except Exception as err:
+                    self._logger.exception(err, stack_info=True)
+                    raise ComponentError(f"HTTPClient Error: {err}") from err
+            ##
+            self.add_metric("FILENAME", results)
+            self._result = results
+            return self._result
+        else:
+            self.url = self.build_url(
+                self.url, args=self.arguments, queryparams=urlencode(self.parameters)
+            )
+            try:
+                if self.use_async is True:
+                    result, error = await self.async_request(
+                        self.url, self.method, data=self._data
+                    )
+                else:
+                    result, error = await self.request(self.url, self.method, data=self._data)
+                if not result:
+                    raise DataNotFound(
+                        f"Data was not found on: {self.url}"
+                    )
+                elif error is not None:
+                    if hasattr(self, "no_errors"):
+                        # check if error is in no_errors
+                        return False
+                    if isinstance(error, BaseException):
+                        raise error
+                    else:
+                        raise ComponentError(f"HTTPClient Error: {error}")
+                # at here, processing Result
+                if self.as_dataframe is True:
+                    try:
+                        result = await self.create_dataframe(result)
+                    except Exception as err:
+                        raise ComponentError(f"RESTClient Error: {err}") from err
+                elif hasattr(self, "download"):
+                    # File downloaded, return same as FileExists
+                    # TODO: error if result is a BytesIO
+                    if isinstance(result, bytes):
+                        self._result = result
+                    elif isinstance(result, PurePath):
+                        file = result
+                        self._logger.debug(f" ::: Checking for File: {file}")
+                        result = {}
+                        if file.exists() and file.is_file():
+                            result[file] = True
+                            self.setTaskVar("DIRECTORY", file.parent)
+                            self.setTaskVar("FILENAME", str(file.name))
+                            self.setTaskVar("FILEPATH", file)
+                            self.add_metric("FILENAME", file)
+                        else:
+                            raise FileNotFound(
+                                f"FileExists: File Doesn't exists: {file}"
+                            )
+                self._result = result
+                return self._result
+            except DataNotFound:
+                raise
+            except Exception as err:
+                self._logger.exception(err, stack_info=True)
+                raise ComponentError(f"HTTPClient: {err}") from err

flowtask/components/ICIMS.py

@@ -0,0 +1,146 @@
+from navconfig.logging import logging
+from querysource.exceptions import DataNotFound as QSNotFound
+from ..exceptions import ComponentError, DataNotFound
+from .QSBase import QSBase
+
+
+class ICIMS(QSBase):
+    """
+    ICIMS
+
+    Overview
+
+        The ICIMS class is a specialized component for handling form data interactions within the ICIMS system.
+        It extends the QSBase class and provides methods for retrieving people data, individual person data,
+        lists of forms, and form data with specific metrics mappings.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+--------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                          |
+        +------------------+----------+--------------------------------------------------------------------------------------+
+        | type             |   Yes    | Defines the type of data handled by the component.                                   |
+        +------------------+----------+--------------------------------------------------------------------------------------+
+        | conditions       |    No    | If any condition is required to do the work.                                         |
+        +------------------+----------+--------------------------------------------------------------------------------------+
+
+    Methods
+
+    people
+
+        Retrieves a list of people data from the ICIMS system.
+        
+        Raises:
+            DataNotFound: If no data is found.
+            ComponentError: If any other error occurs during execution.
+
+    person
+
+        Retrieves individual person data from the ICIMS system and adds a person_id to the data.
+        
+        Raises:
+            DataNotFound: If no data is found.
+            ComponentError: If any other error occurs during execution.
+
+    forms_list
+
+        Retrieves a list of forms from the ICIMS system.
+        
+        Raises:
+            DataNotFound: If no data is found.
+            ComponentError: If any other error occurs during execution.
+
+    form_data
+
+        Retrieves form data from the ICIMS system and maps internal metrics to output names.
+        
+        Raises:
+            DataNotFound: If no data is found.
+            ComponentError: If any other error occurs during execution.
+
+    Return
+
+    The methods in this class return the requested data from the ICIMS system, formatted according to the
+    specific requirements of each method.
+
+    
+
+        Example:
+
+        ```yaml
+        ICIMS:
+          type: people
+        ```
+
+    """
+
+    type = "form_data"
+    _driver = "icims"
+    _metrics: dict = {
+        "requestedby": "requested_by",
+        "updatedby": "updated_by",
+        "formname": "form_name",
+        "updateddate": "updated_date",
+        "completeddate": "completed_date",
+        "completedby": "completed_by",
+    }
+
+    async def people(self):
+        try:
+            row = await self._qs.people()
+            if row:
+                return row
+            else:
+                return []
+        except QSNotFound as err:
+            raise DataNotFound(f"ICIMS Not Found: {err}") from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(f"ICIMS ERROR: {err!s}") from err
+
+    async def person(self):
+        try:
+            row = await self._qs.person()
+            if row:
+                row["person_id"] = self._variables["person_id"]
+                return [row]
+            else:
+                return []
+        except QSNotFound as err:
+            raise DataNotFound(f"ICIMS Not Found: {err}") from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(f"ICIMS ERROR: {err!s}") from err
+
+    async def forms_list(self):
+        try:
+            row = await self._qs.forms_list()
+            if row:
+                return row["forms"]
+            else:
+                return []
+        except QSNotFound as err:
+            raise DataNotFound(f"ICIMS Not Found: {err}") from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(f"ICIMS ERROR: {err!s}") from err
+
+    async def form_data(self):
+        try:
+            row = await self._qs.form_data()
+            if row:
+                for key, name in self._metrics.items():
+                    try:
+                        row[name] = row[key]
+                        del row[key]
+                    except (TypeError, KeyError):
+                        pass
+                return [row]
+            else:
+                return []
+        except QSNotFound as err:
+            raise DataNotFound(f"ICIMS Not Found: {err}") from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(f"ICIMS ERROR: {err!s}") from err

flowtask/components/IF.py

@@ -0,0 +1,179 @@
+from collections.abc import Callable
+from jsonpath_ng import parse
+import asyncio
+import pandas as pd
+from ..exceptions import ComponentError
+from .BaseLoop import BaseLoop
+
+
+class IF(BaseLoop):
+    """
+    IF Component
+
+    Executes one of two components based on a condition.
+
+    true_component: The component to execute if the condition evaluates to True.
+    false_component: The component to execute if the condition evaluates to False.
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._condition = kwargs.get("condition", None)
+        self._true_component = kwargs.get("true_component", None)
+        self._false_component = kwargs.get("false_component", None)
+        super(IF, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """
+        Initialize the component.
+        """
+        await super(IF, self).start(**kwargs)
+        # Ensure the condition and components are defined
+        if not self._condition:
+            raise ComponentError(
+                "The 'condition' argument must be provided."
+            )
+        if not self._true_component or not self._false_component:
+            raise ComponentError(
+                "Both 'true_component' and 'false_component' must be specified."
+            )
+        return True
+
+    def _evaluate_condition(self, condition: str) -> bool:
+        """
+        Evaluate a condition based on the type of self.data.
+
+        Args:
+            condition (str): A condition in the format "<jsonpath> <operator> <value>".
+
+        Returns:
+            bool: True if the condition is satisfied, else False.
+        """
+        if isinstance(self.data, dict):
+            try:
+                jsonpath_expr, operator, expected_value = self._parse_condition(condition)
+                jsonpath = parse(jsonpath_expr)
+                matches = jsonpath.find(self.data)
+                actual_value = matches[0].value if matches else None
+                return self._evaluate_operator(
+                    actual_value,
+                    operator,
+                    expected_value
+                )
+            except Exception as err:
+                raise ComponentError(
+                    f"Error evaluating condition '{condition}': {err}"
+                )
+
+        elif isinstance(self.data, pd.DataFrame):
+            try:
+                filtered = self.data.query(condition)
+                return not filtered.empty
+            except Exception as err:
+                raise ComponentError(
+                    f"Error evaluating DataFrame condition '{condition}': {err}"
+                )
+
+        elif isinstance(self.data, str):
+            return condition in self.data
+
+        else:
+            raise ComponentError(
+                f"Unsupported data type for condition evaluation: {type(self.data)}"
+            )
+
+    def _parse_condition(self, condition: str):
+        """
+        Parse the condition into JSONPath, operator, and expected value.
+
+        Args:
+            condition (str): A condition string, e.g., "$.metadata.type == 'recapDefinition'".
+
+        Returns:
+            tuple: (jsonpath_expr, operator, expected_value)
+        """
+        operators = ["==", "!=", ">", "<", ">=", "<="]
+        for op in operators:
+            if op in condition:
+                parts = condition.split(op)
+                if len(parts) != 2:
+                    raise ValueError(
+                        f"Invalid condition format: {condition}"
+                    )
+                jsonpath_expr, expected_value = parts
+                expected_value = expected_value.strip().strip("'\"")  # Remove quotes
+                return jsonpath_expr.strip(), op, expected_value
+        raise ValueError(
+            f"No valid operator found in condition: {condition}"
+        )
+
+    def _evaluate_operator(self, actual_value, operator, expected_value):
+        """
+        Evaluate the comparison operator.
+
+        Args:
+            actual_value: The value extracted from JSONPath.
+            operator: The comparison operator as a string.
+            expected_value: The value to compare against.
+
+        Returns:
+            bool: True if the comparison is satisfied, else False.
+        """
+        if operator == "==":
+            return actual_value == expected_value
+        elif operator == "!=":
+            return actual_value != expected_value
+        elif operator == ">":
+            return actual_value > expected_value
+        elif operator == "<":
+            return actual_value < expected_value
+        elif operator == ">=":
+            return actual_value >= expected_value
+        elif operator == "<=":
+            return actual_value <= expected_value
+        else:
+            raise ValueError(
+                f"Unsupported operator: {operator}"
+            )
+
+    async def run(self):
+        """
+        Executes the appropriate component based on the condition.
+        """
+        try:
+            condition_result = self._evaluate_condition(self._condition)
+        except ComponentError as e:
+            raise ComponentError(
+                f"Error evaluating condition '{self._condition}': {e}"
+            )
+
+        selected_component = (
+            self._true_component if condition_result else self._false_component
+        )
+
+        step, idx = self._TaskPile.getStepByName(selected_component)
+        if step is None:
+            raise ComponentError(
+                f"Component '{selected_component}' not found in Task Definition."
+            )
+
+        target, params, stat = self.get_component(step)
+        component_instance = self.create_component(
+            target,
+            value=self.data,
+            stat=stat,
+            **params
+        )
+        self._result = await self.exec_component(
+            component_instance,
+            step.name
+        )
+        # Remove executed components
+        self._TaskPile.delStepByName(self._true_component)
+        self._TaskPile.delStepByName(self._false_component)
+        return self._result