flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/DateList.py

@@ -0,0 +1,255 @@
+import asyncio
+import logging
+import datetime
+import builtins
+from numbers import Number
+from collections.abc import Callable
+from asyncdb.exceptions import NoDataFound, ProviderError
+from querysource.exceptions import DataNotFound
+from querysource.utils.functions import format_date
+import querysource.utils.functions as qsfunctions
+from ..exceptions import ComponentError, NotSupported
+from ..utils import fnExecutor
+from .IteratorBase import IteratorBase
+
+
+class DateList(IteratorBase):
+    """
+    DateList.
+
+    Overview
+
+            Run as many times as dates generated by this component.
+    .. table:: Properties
+       :widths: auto
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | range        |   Yes    | Start and end date range parameter                                |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | inc          |   Yes    | Increases the counter value N times                               |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | format       |   Yes    | Date format to generate                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | vars         |   Yes    | Generate variables from the start date you generate in each       |
+    |              |          | iterator                                                          |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+
+    Return the list of arbitrary days
+
+
+
+        Example:
+
+        ```yaml
+        DateList:
+          range:
+            start: 01-12-2021
+            end: 24-12-2021
+            format: '%d-%m-%Y'
+          inc: 1
+          format: '%Y-%m-%dT%H:%M:%S'
+          vars:
+            firstdate: '{date}'
+            lastdate:
+            - to_midnight
+            - mask: '%Y-%m-%d %H:%M:%S'
+          iterate: true
+        ```
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._path = None
+        self.pattern = None
+        self._increment = 1
+        self._iterator = None
+        self._format = "%Y-%m-%d"
+        super(DateList, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Get Range of Dates."""
+        await super(DateList, self).start()
+        if not hasattr(self, "range"):
+            raise ComponentError("Error: you need to specify a range attribute")
+        # get start and end dates
+        if "start" not in self.range:
+            raise ComponentError("Error: you need to specify a start Date")
+        if "end" not in self.range:
+            raise ComponentError("Error: you need to specify an End Date")
+        ### Incremental:
+        if hasattr(self, "inc"):
+            self._increment = self.inc
+        if "increment" in self._variables:
+            self._increment = int(self._variables["increment"])
+        elif "inc" in self._args:
+            self._increment = int(self._args["inc"])
+        self.setVar("inc", self._increment)
+        if hasattr(self, "format"):
+            self._format = self.format
+        # set the start:end of the range
+        if "start" in self._args:
+            range_start = self._args["start"]
+        elif isinstance(self.range["start"], list):
+            range_start = fnExecutor(self.range["start"])
+        else:
+            range_start = self.range["start"]
+        if "end" in self._args:
+            range_end = self._args["end"]
+        elif isinstance(self.range["end"], list):
+            range_end = fnExecutor(self.range["end"])
+        else:
+            range_end = self.range["end"]
+        # build the dates
+        start = datetime.datetime.strptime(range_start, self.range["format"])
+        end = datetime.datetime.strptime(range_end, self.range["format"])
+        if self._debug:
+            self._logger.debug(f"RANGE> {start!s} - {end!s}")
+            self._logger.debug(f'INCREMENT> {self._increment}')
+        # define the generator
+        self._iterator = (
+            start + datetime.timedelta(days=x)
+            for x in range(0, (end - start).days + 1, self._increment)
+        )
+        self.add_metric("RANGE", f"{start!s} - {end!s}")
+        self.add_metric("INCREMENT", self._increment)
+        return True
+
+    def createJob(self, target, params, dt, d):
+        """Create the Job Component."""
+        self._result = str(dt)
+        if hasattr(self, "vars"):
+            for name, value in self.vars.items():
+                kwargs = {}
+                if value == "{date}":
+                    val = value.format(date=dt)
+                elif name in self._params:
+                    val = self._params[name]
+                elif name in self.conditions:
+                    val = self.conditions[name]
+                elif isinstance(value, list):
+                    fname = value[0]
+                    try:
+                        kwargs = value[1].copy()
+                        for k, v in value[1].items():
+                            if isinstance(v, int):
+                                kwargs[k] = v
+                            elif "{increment}" in v:
+                                kwargs[k] = self._increment - 1
+                            elif "{date}" in v:
+                                kwargs[k] = v.format(date=dt)
+                    except IndexError:
+                        kwargs = {}
+                    if 'value' not in kwargs:
+                        t = {"value": d}
+                    else:
+                        t = {}
+                    kwargs = {**kwargs, **t}
+                    try:
+                        f = getattr(qsfunctions, fname)
+                    except AttributeError:
+                        f = getattr(builtins, fname)
+                    try:
+                        val = f(**kwargs)
+                    except (TypeError, AttributeError):
+                        try:
+                            val = globals()[fname](**kwargs)
+                        except (TypeError, ValueError) as err:
+                            print(str(err))
+                            val = None
+                elif name in self._mask:
+                    val = self._mask[name]
+                elif isinstance(value, int):
+                    val = int(value)
+                elif isinstance(value, Number):
+                    val = value
+                elif isinstance(value, str):
+                    val = value.format(date=dt)
+                else:
+                    val = value.format(date=dt)
+                params[name] = val
+                self.setVar(name, val)
+        else:
+            # think if we need to create the vars using self._vars
+            for name, value in self._vars.items():
+                print("VARIABLES::: ", name, value)
+                if value == "{date}":
+                    params[name] = dt
+                    self.setVar(name, dt)
+                else:
+                    params[name] = value
+                    self.setVar(name, value)
+        params["date"] = dt
+        params["filterdate"] = dt
+        return self.get_job(target, **params)
+
+    def setAttributes(self, dt):
+        """Defining the result to component."""
+        try:
+            self.setVar("{}_{}".format(self.StepName, "DATE"), dt)
+            self._result = str(dt)
+        except Exception as err:
+            print(err)
+            return False
+
+    async def close(self, job=None):
+        close = getattr(job, "close", None)
+        if job:
+            if asyncio.iscoroutinefunction(close):
+                await job.close()
+            else:
+                job.close()
+
+    async def run(self):
+        """Async Run Method."""
+        status = False
+        if not self._iterator:
+            return False
+        if self.iterate:
+            # iterate over next task
+            step, target, params = self.get_step()
+            step_name = step.name
+            for d in self._iterator:
+                dt = format_date(d, self._format)
+
+                self.setAttributes(dt)
+                job = self.createJob(target, params, dt, d)
+                if job:
+                    try:
+                        status = await self.async_job(job, step_name)
+                    except (NoDataFound, DataNotFound) as err:
+                        # its a data component a no data was found
+                        logging.debug(
+                            f"Data not Found for Task {step_name}, Error: {err}"
+                        )
+                        continue
+                    except (ProviderError, ComponentError) as err:
+                        raise ComponentError(
+                            f"Error running Component {step_name}, error: {err}"
+                        ) from err
+                    except NotSupported as err:
+                        raise NotSupported(
+                            f"Not Supported: Error on Component {step_name}, error: {err}"
+                        ) from err
+                    except Exception as err:
+                        raise ComponentError(
+                            f"DateList: Component Error on {step_name}, error: {err}"
+                        ) from err
+                    finally:
+                        await self.close(job)
+            if status is False:
+                return False
+            else:
+                return True
+        else:
+            # return the list of date
+            self._result = list(self._iterator)
+            return True

flowtask/components/DbClient.py

@@ -0,0 +1,163 @@
+from abc import ABC
+from typing import Union, Optional
+from pathlib import PurePath
+import aiofiles
+import pandas
+from asyncdb.drivers.base import BaseDriver
+from asyncdb.exceptions import NoDataFound
+from navconfig.logging import logging
+from ..utils import cPrint
+from ..utils.functions import is_empty, as_boolean
+from ..exceptions import ComponentError, FileError, DataNotFound
+from ..interfaces import DBInterface
+
+
+class DbClient(DBInterface):
+    """
+    DbClient.
+
+    Abstract base class for database clients using AsyncDB.
+
+    Provides common methods for connecting to a database, executing queries,
+    and handling results.
+
+    Inherits from the `DBInterface` interface.
+
+    .. note::
+    This class is intended to be subclassed by specific database client implementations.
+
+    .. table:: Properties
+    :widths: auto
+
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | Name               | Required | Description                                                                    |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | driver             |   Yes    | Database driver to use (e.g., "pg" for PostgreSQL).                            |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | credentials        |   Yes    | Dictionary containing database connection credentials (user, password, etc.).  |
+    |                    |          | Defined by the `_credentials` class attribute.                                 |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | raw_result         |    No    | Boolean flag indicating whether to return raw query results or convert them    |
+    |                    |          | to dictionaries (defaults to False).                                           |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | infer_types        |    No    | Boolean flag indicating whether to infer data types for pandas DataFrames      |
+    |                    |          | created from query results (defaults to False).                                |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | as_dataframe       |    No    | Boolean flag indicating whether to convert query results to a pandas DataFrame |
+    |                    |          | (defaults to True).                                                            |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+    | as_string          |    No    | Boolean flag indicating whether to convert object columns in DataFrames        |
+    |                    |          | to strings (defaults to False). Converting to strings requires type            |
+    |                    |          | inference (set `infer_types` to True if used).                                 |
+    +--------------------+----------+--------------------------------------------------------------------------------+
+
+    .. returns::
+    varies
+        The return value of the `_query` method depends on the `raw_result` and `as_dataframe` properties:
+            * If `raw_result` is True: Returns the raw result object specific to the database driver.
+            * If `raw_result` is False and `as_dataframe` is True: Returns a pandas DataFrame containing the query results.
+            * If `raw_result` is False and `as_dataframe` is False: Returns a list of dictionaries representing the query results.
+        Raises exceptions for errors during execution (e.g., `DataNotFound`, `ComponentError`).
+    """  # noqa
+    _credentials = {
+        "user": str,
+        "password": str,
+        "host": str,
+        "port": int,
+        "database": str,
+    }
+
+    def __init__(
+        self,
+        *args,
+        **kwargs
+    ) -> None:
+        self.raw_result: bool = kwargs.pop("raw_result", False)
+        self.infer_types: bool = as_boolean(kwargs.pop("infer_types", False))
+        self.as_dataframe: bool = as_boolean(kwargs.pop("as_dataframe", True))
+        # using "string" instead objects in pandas
+        self.as_string: bool = as_boolean(kwargs.pop("as_string", False))
+        super().__init__(*args, **kwargs)
+        if self.as_string is True:
+            self.infer_types = True
+        # any other argument
+        self._args = kwargs
+
+    async def open_sqlfile(self, file: PurePath, **kwargs) -> str:
+        if file.exists() and file.is_file():
+            content = None
+            # open SQL File:
+            async with aiofiles.open(file, "r+") as afp:
+                content = await afp.read()
+                # check if we need to replace masks
+            if hasattr(self, "masks"):
+                content = self.mask_replacement(content)
+            if self.use_template is True:
+                content = self._templateparser.from_string(content, kwargs)
+            return content
+        else:
+            raise FileError(
+                f"{__name__}: Missing SQL File: {file}"
+            )
+
+    async def close(self, connection: BaseDriver = None) -> None:
+        if not connection:
+            connection = self._connection
+        try:
+            if connection is not None:
+                await connection.close()
+        except Exception as err:  # pylint: disable=W0703
+            logging.error(
+                f"DbClient Closing error: {err}"
+            )
+
+    async def _query(
+        self, query: Union[str, dict], connection: BaseDriver = None
+    ) -> Union[list, dict]:
+        if not connection:
+            connection = self._connection
+        try:
+            if isinstance(query, dict):
+                result, error = await connection.query(**query)
+                print('RESULT > ', result, error)
+            else:
+                result, error = await connection.query(query)
+            if error:
+                raise ComponentError(
+                    f"DbClient: Query Error: {error}"
+                )
+            if self.raw_result is True:
+                return result
+            else:  # converting to dict
+                result = [dict(row) for row in result]
+                return result
+        except (DataNotFound, NoDataFound) as err:
+            raise DataNotFound("DbClient: Data not found") from err
+        except Exception as err:
+            raise ComponentError(f"{err}") from err
+
+    async def get_dataframe(self, result):
+        try:
+            df = pandas.DataFrame(result)
+        except Exception as err:  # pylint: disable=W0703
+            logging.exception(err, stack_info=True)
+        # Attempt to infer better dtypes for object columns.
+        if is_empty(df):
+            raise DataNotFound("DbClient: Data not Found")
+        df.infer_objects()
+        if self.infer_types is True:
+            df = df.convert_dtypes(convert_string=self.as_string)
+        if self._debug is True:
+            cPrint("Data Types:")
+            print(df.dtypes)
+        if hasattr(self, "drop_empty"):
+            df.dropna(axis=1, how="all", inplace=True)
+            df.dropna(axis=0, how="all", inplace=True)
+        if hasattr(self, "dropna"):
+            df.dropna(subset=self.dropna, how="all", inplace=True)
+        if (
+            hasattr(self, "clean_strings") and getattr(self, "clean_strings", False) is True
+        ):
+            u = df.select_dtypes(include=["object", "string"])
+            df[u.columns] = u.fillna("")
+        return df

flowtask/components/DialPad.py

@@ -0,0 +1,146 @@
+import asyncio
+import pandas as pd
+from collections.abc import Callable
+from urllib.parse import urljoin
+from io import StringIO
+from navconfig.logging import logging
+from .flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError
+
+class DialPad(FlowComponent, HTTPService):
+    """
+        DialPad
+
+        Overview
+
+            The DialPad class is a component for interacting with the DialPad API. It extends the FlowComponent and HTTPService
+            classes, providing methods for authentication, fetching statistics, and handling API responses.
+
+        .. table:: Properties
+        :widths: auto
+
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | Name             | Required | Description                                                                                      |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | accept           |   No     | The accepted content type for API responses, defaults to "application/json".                     |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | download         |   No     | The download flag indicating if a file download is required.                                     |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | _credentials     |   Yes    | A dictionary containing the API key for authentication.                                          |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | _base_url        |   Yes    | The base URL for the DialPad API.                                                                |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | auth             |   Yes    | The authentication header for API requests.                                                      |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+        Return
+
+            The methods in this class manage the interaction with the DialPad API, including initialization, fetching statistics,
+            processing results, and handling credentials.
+
+    """ # noqa
+    accept: str = "text/csv"
+    download = None
+    _credentials: dict = {"APIKEY": str}
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        FlowComponent.__init__(self, loop=loop, job=job, stat=stat, **kwargs)
+        HTTPService.__init__(self, **kwargs)
+
+    async def start(self, **kwargs):
+        self._base_url = "https://dialpad.com/api/v2/"
+
+        self.processing_credentials()
+        self.auth = {"apikey": self.credentials["APIKEY"]}
+
+        if self.previous:
+            self.data = self.input
+        else:
+            self.data = None
+        return True
+
+    async def dialpad_stats(self):
+        # processing statistics asynchronously
+        self.accept: str = "application/json"
+        stats_url = urljoin(self._base_url, "stats/")
+        processing_result, _ = await self.session(
+            stats_url, "post", data=self.body_params, use_json=True
+        )
+        request_id = processing_result.get("request_id")
+
+        count = 0
+        while True:
+            get_result_url = urljoin(stats_url, request_id)
+            response_result, _ = await self.session(get_result_url, use_json=True)
+            file_url = response_result.get("download_url", None)
+            if file_url or count > 60:
+                break
+            count += 1
+            logging.debug(f"Try: {count!s}")
+            await asyncio.sleep(10)
+
+        self.accept: str = "text/csv"
+        self.download = False
+        result, _ = await self.session(file_url)
+        df_results = await self.from_csv(StringIO(result))
+        return df_results
+
+    async def dialpad_send_sms(self):
+        # send SMS message
+        url = urljoin(self._base_url, "sms")
+        from_number = self.body_params.get("from_number")
+        to_numbers = self.body_params.get("to_numbers")
+        text = self.body_params.get("text")
+
+        if self.data is not None and not self.data.empty:
+            if text in self.data.columns:
+                text = self.data.get(text).iloc[0]
+            if from_number in self.data.columns:
+                from_number = self.data.get(from_number).iloc[0]
+            if not isinstance(to_numbers, list):
+                if isinstance(to_numbers, str):
+                    if to_numbers in self.data.columns:
+                        to_numbers = self.data[to_numbers].tolist()
+                    else:
+                        to_numbers = [to_numbers]
+
+        payload = {
+            "infer_country_code": True,
+            "text": text,
+            "from_number": from_number,
+            "to_numbers": to_numbers
+        }
+
+        self.accept: str = "application/json"
+        stats_url = urljoin(self._base_url, "sms")
+        result, _ = await self.session(
+            stats_url, "post", data=payload, use_json=True
+        )
+
+        df_results = pd.DataFrame([result])
+        return df_results
+
+    async def run(self):
+        try:
+            method = getattr(self, f"dialpad_{self.type}")
+        except AttributeError as ex:
+            raise ComponentError(f"{__name__}: Wrong 'type' on task definition") from ex
+        result = await method()
+        self._result = result
+        print(self._result)
+        if self._debug:
+            columns = list(self._result.columns)
+            print(f"Debugging: {self.__name__} ===")
+            for column in columns:
+                t = self._result[column].dtype
+                print(column, "->", t, "->", self._result[column].iloc[0])
+        return self._result
+
+    async def close(self):
+        pass