flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/QueryToPandas.py

@@ -0,0 +1,432 @@
+import logging
+import asyncio
+from pathlib import Path, PurePath
+from collections.abc import Callable
+from pprint import pprint
+import aiofiles
+import urllib3
+import pandas as pd
+from asyncdb.exceptions import NoDataFound
+from querysource.types.validators import Entity
+from ..exceptions import (
+    ComponentError,
+    DataNotFound,
+    NotSupported,
+    FileError
+)
+from ..interfaces.qs import QSSupport
+from ..utils import cPrint, is_empty, SafeDict
+from ..conf import TASK_PATH
+from .flow import FlowComponent
+from ..interfaces import TemplateSupport
+
+urllib3.disable_warnings()
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+
+class QueryToPandas(FlowComponent, TemplateSupport, QSSupport):
+    """
+    QueryToPandas.
+
+    Overview
+
+        This component fetches data using QuerySource and transforms it into a Pandas DataFrame.
+
+    .. table:: Properties
+    :widths: auto
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | query        |   Yes    | Represents an SQL query                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | query_slug   |   Yes    | Named queries that are saved in Navigator (QuerySource)           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | as_dict      |   Yes    | True | False. if true, it returns the data in JSON format         |
+    |              |          | instead of a dataframe                                            |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | raw_result   |   Yes    | Returns the data in the NATIVE FORMAT of the database for         |
+    |              |          | example ( pg RECORDSET)                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | file_sql     |   Yes    | SQL comes from a sql file                                         |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | use_template |   Yes    | The component is passed to the SQL file through a  template       |
+    |              |          | replacement system                                                |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | infer_types  |   Yes    | Type inference, give the component the power to decide the data   |
+    |              |          | types of each column                                              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | drop_empty   |   Yes    | False | True  delete (drop) any column that is empty              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | dropna       |   Yes    | False | True  delete all NA (Not a Number)                        |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | fillna       |   Yes    | False | True  fills with an EMPTY SPACE all the NAs of the        |
+    |              |          | dataframe                                                         |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | clean_strings|   Yes    | Fills with an empty space the NA,but ONLY of the fields of        |
+    |              |          | type string                                                       |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | clean_dates  |   Yes    | Declares NONE any date field that has a NAT (Not a Time)          |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | conditions   |   Yes    | This attribute allows me to apply conditions to filter the data   |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | formit       |   Yes    | Form id   (i have doubts about this)                              |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | orgid        |   Yes    | Organization id     (i have doubts about this)                    |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | refresh      |   Yes    | Refreshes the data in the QueryToPandas                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | to_string    |   No     | Whether to convert all data to string type. Default is True.      |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | as_objects   |   No     | Whether to return the result as objects. Default is True.         |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | datatypes    |   No     | A dictionary specifying data types for columns.                   |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | datasource   |   No     | The datasource to fetch the data from. Default is "db".           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    Returns
+
+    This component returns a Pandas DataFrame if the query is successfully executed and data is fetched,
+    otherwise raises a ComponentError.
+
+
+
+        Example:
+
+        ```yaml
+        QueryToPandas:
+          query: SELECT formid, orgid FROM banco.forms WHERE enabled = true
+        ```
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.data = None
+        self.infer_types: bool = True
+        self.to_string: bool = True
+        self._query: dict = {}
+        self.as_dict: bool = False
+        self.as_objects: bool = True
+        self._dtypes: dict = {}
+        self.datatypes: dict = {}
+        self.use_template: bool = kwargs.get('use_template', False)
+        self._driver = kwargs.get("driver", "pg")
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def open_sqlfile(self, file: PurePath, **kwargs) -> str:
+        if file.exists() and file.is_file():
+            content = None
+            # open SQL File:
+            async with aiofiles.open(file, "r+") as afp:
+                content = await afp.read()
+                # check if we need to replace masks
+            if hasattr(self, "masks"):
+                self._logger.debug(f"QueryToPandas Masks: {self.masks}")
+                if "{" in content:
+                    content = self.mask_replacement(content)
+            if self.use_template is True:
+                content = self._templateparser.from_string(content, kwargs)
+            elif hasattr(self, "conditions"):
+                content = self.conditions_replacement(content)
+            return content
+        else:
+            raise FileError(f"{self.__name__}: Missing SQL File: {file}")
+
+    async def start(self, **kwargs):
+        await super(QueryToPandas, self).start(**kwargs)
+        # compute conditions:
+        self.set_conditions()
+        # check if sql comes from a filename:
+        if hasattr(self, "file_sql") or hasattr(self, "query_file"):
+            # based on a list/dict of queries
+            if hasattr(self, "file_sql"):
+                query = self.file_sql
+            else:
+                query = self.query_file
+            if isinstance(query, PurePath):
+                self._query = []
+                if query.exists() and query.is_file():
+                    sql = await self.open_sqlfile(query)
+                    self._query.append(sql)
+            elif isinstance(query, str):
+                self._query = []
+                try:
+                    file_path = Path(TASK_PATH).joinpath(self._program, "sql", query)
+                    if file_path.exists() and file_path.is_file():
+                        sql = await self.open_sqlfile(file_path)
+                        self._query.append(sql)
+                except Exception as ex:
+                    raise FileError(f"File SQL doesn't exists: {query!s}: {ex}") from ex
+            elif isinstance(query, list):  # list of queries
+                self._query = []
+                for file_sql in query:
+                    file_path = Path(TASK_PATH).joinpath(self._program, "sql", file_sql)
+                    if file_path.exists() and file_path.is_file():
+                        sql = await self.open_sqlfile(file_path)
+                        self._query.append(sql)
+            elif isinstance(query, dict):  # named queries
+                self._query = {}
+                for key, file_sql in query.items():
+                    file_path = Path(TASK_PATH).joinpath(self._program, "sql", file_sql)
+                    if file_path.exists() and file_path.is_file():
+                        sql = await self.open_sqlfile(file_path)
+                        self._query[key] = sql
+        elif hasattr(self, "query_slug"):
+            if isinstance(self.query_slug, str):  # pylint: disable=E0203
+                if "{" in self.query_slug:  # pylint: disable=E0203 # noqa
+                    self.query_slug = self.mask_replacement(self.query_slug)
+                self._query[self.query_slug] = self.query_slug
+            elif isinstance(self.query_slug, list):
+                for slug in self.query_slug:
+                    self._query[slug] = slug
+            elif isinstance(self.query_slug, dict):
+                # iterate over all conditions and search in masks:
+                for key, data in self.query_slug.items():
+                    slug = data["slug"]
+                    for mask, replace in self._mask.items():
+                        if mask in data["conditions"]:
+                            self.query_slug[key]["conditions"][mask] = replace
+                    self._query[key] = slug
+        elif hasattr(self, "query"):
+            if isinstance(self.query, str):  # pylint: disable=E0203
+                self._query = []
+                if hasattr(self, "masks"):
+                    self.query = self.mask_replacement(self.query)
+                elif "{" in self.query and hasattr(self, "conditions"):
+                    try:
+                        self.query = self.query.format(**self.conditions)
+                    except Exception as err:
+                        self._logger.warning(f"Error replacing Vars in Query: {err}")
+                try:
+                    self.query = self.query.format(**self._variables)
+                except Exception as err:
+                    self._logger.warning(f"Error replacing Vars in Query: {err}")
+                self._query.append(self.query)
+            elif isinstance(self.query, dict):  # named queries
+                self._query = {}
+                for key, query in self.query.items():
+                    query = self.mask_replacement(query)
+                    try:
+                        query = query.format(**self._variables)
+                    except Exception:
+                        pass
+                    self._query[key] = query
+            elif isinstance(self.query, list):  # need to be concatenated
+                self._query = []
+                for query in self.query:
+                    query = self.mask_replacement(query)
+                    try:
+                        for val in self._variables:
+                            if isinstance(self._variables[val], list):
+                                result = ", ".join(self._variables[val])
+                            else:
+                                result = ", ".join(
+                                    "'{}'".format(v) for v in self._variables[val]
+                                )
+                        query = query.format(**self._variables)
+                    except Exception:
+                        pass
+                    self._query.append(query)
+        if hasattr(self, "conditions"):
+            self.set_conditions("conditions")
+            cPrint("NEW CONDITIONS ARE> ", level="WARN")
+            pprint(self.conditions)
+
+        # Replace variables
+        if isinstance(self._query, list):
+            queries = []
+            for query in self._query:
+                values = {}
+                for key, val in self._variables.items():
+                    if isinstance(val, list):
+                        value = ", ".join(
+                            "'{}'".format(Entity.quoteString(v)) for v in val
+                        )
+                    else:
+                        value = val
+                    query = query.replace("{{{}}}".format(str(key)), str(value))
+                    values[key] = value
+                # using safeDict
+                query.format_map(SafeDict(**values))
+                queries.append(query)
+            self._query = queries
+        return True
+
+    async def close(self):
+        """Method."""
+
+    async def run(self):
+        # TODO: support for datasources
+        # TODO: using maps to transform data types
+        if not self._query:
+            raise ComponentError(
+                "QueryToPandas: Empty Query/Slug or File"
+            )
+        if hasattr(self, "query") or hasattr(self, "file_sql"):
+            try:
+                connection = await self.create_connection()
+            except Exception as err:
+                self._logger.exception(err, stack_info=True)
+                raise
+            if isinstance(self._query, list):  # list of queries
+                results = []
+                async with await connection.connection() as conn:
+                    for query in self._query:
+                        try:
+                            res, error = await conn.query(query)
+                            if error:
+                                raise DataNotFound(error)
+                            result = [dict(row) for row in res]
+                        except NoDataFound:
+                            result = []
+                        except Exception as err:
+                            self._logger.error(err)
+                            raise
+                        ln = len(result)
+                        st = {
+                            "result": ln
+                        }
+                        self.add_metric("Query", st)
+                        if ln == 1 and self.as_dict is True:
+                            # saving only one row
+                            result = dict(result[0])
+                            results.append(result)
+                        else:
+                            results.extend(result)
+                if hasattr(self, "raw_result"):
+                    self._result = results
+                    self._variables[f"{self.StepName}_NUMROWS"] = len(results)
+                else:
+                    self._result = await self.get_dataframe(
+                        results, infer_types=self.infer_types
+                    )
+                    numrows = len(self._result.index)
+                    self._variables[f"{self.StepName}_NUMROWS"] = numrows
+            elif isinstance(self._query, dict):  # Named queries
+                self._result = {}
+                results = []
+                async with await connection.connection() as conn:
+                    for key, query in self._query.items():
+                        try:
+                            res, error = await conn.query(query)
+                            if error:
+                                raise DataNotFound(error)
+                            result = [dict(row) for row in res]
+                        except NoDataFound:
+                            result = []
+                        except Exception as err:
+                            self._logger.error(err)
+                            raise
+                        ln = len(result)
+                        st = {"query": key, "result": ln}
+                        self.add_metric("Query", st)
+                        if ln == 1:
+                            # saving only one row
+                            result = dict(result[0])
+                        if hasattr(self, "raw_result"):
+                            self._result[key] = result
+                            self._variables[f"{self.StepName}_{key}_NUMROWS"] = len(
+                                result
+                            )
+                        else:
+                            df = await self.get_dataframe(
+                                result, infer_types=self.infer_types
+                            )
+                            self._result[key] = df
+                            self._variables[f"{self.StepName}_{key}_NUMROWS"] = len(
+                                df.index
+                            )
+            else:
+                raise NotSupported(f"{self.__name__}: Incompatible Query Method.")
+        elif hasattr(self, "query_slug"):
+            # TODO: assign the datasource to QuerySource connection
+            self.add_metric("SLUG", self.query_slug)
+            if isinstance(self.query_slug, dict):
+                # return a list of queries
+                self._result = {}
+                for key, data in self.query_slug.items():
+                    slug = data["slug"]
+                    conditions = data["conditions"]
+                    try:
+                        result = await self.get_qs(slug, conditions)
+                        ln = len(result)
+                        st = {"query": key, "result": ln}
+                        if ln == 1 and self.as_dict is True:
+                            # saving only one row
+                            result = dict(result[0])
+                    except (DataNotFound, NoDataFound) as ex:
+                        raise DataNotFound(str(ex)) from ex
+                    if hasattr(self, "raw_result"):
+                        self._result[key] = result
+                        self._variables[f"{self.StepName}_{key}_NUMROWS"] = len(result)
+                        self.add_metric("NUMROWS", len(result))
+                    else:
+                        df = await self.get_dataframe(
+                            result, infer_types=self.infer_types
+                        )
+                        self._result[key] = df
+                        self._variables[f"{self.StepName}_{key}_NUMROWS"] = len(
+                            df.index
+                        )
+                        self.add_metric("NUMROWS", len(df.index))
+            else:
+                results = []
+                for key, slug in self._query.items():
+                    conditions = {}
+                    if hasattr(self, "conditions"):
+                        conditions = self.conditions
+                    try:
+                        result = await self.get_qs(slug, conditions)
+                        ln = len(result)
+                        self._logger.debug(f"QS {key}: length: {ln}")
+                        st = {"query": key, "result": ln}
+                        if ln == 1 and self.as_dict is True:
+                            # saving only one row
+                            result = dict(result[0])
+                    except (DataNotFound, NoDataFound):
+                        result = {}
+                    except Exception as err:
+                        self._logger.exception(err, stack_info=False)
+                        raise
+                    results.extend(result)
+                if hasattr(self, "raw_result"):
+                    self._result = results
+                    self._variables[f"{self.StepName}_NUMROWS"] = len(results)
+                    self.add_metric("NUMROWS", len(result))
+                else:
+                    self._result = await self.get_dataframe(
+                        results, infer_types=self.infer_types
+                    )
+                    numrows = len(self._result.index)
+                    self._variables[f"{self.StepName}_NUMROWS"] = numrows
+                    self.add_metric("NUMROWS", numrows)
+        else:
+            raise NotSupported(f"{self.__name__}: Method not allowed")
+        if is_empty(self._result):
+            raise DataNotFound(f"{self.__name__}: Data Not Found")
+        else:
+            ### making traspose of data:
+            if hasattr(self, "transpose"):
+                # transpose rows to columns:
+                # self._result = self._result.transpose()
+                self._result = pd.melt(self._result, id_vars=self.transpose["columns"])
+                if "variable" in self.transpose:
+                    # rename variable to a new name:
+                    self._result.rename(
+                        columns={"variable": self.transpose["variable"]}, inplace=True
+                    )
+                if "value" in self.transpose:
+                    self._result.rename(
+                        columns={"value": self.transpose["value"]}, inplace=True
+                    )
+            if self._debug is True:
+                print("== DATA PREVIEW ==")
+                print(self._result)
+                print()
+            return self._result

flowtask/components/RESTClient.py

@@ -0,0 +1,195 @@
+"""RESTClient.
+
+Basic component for making RESTful queries to URLs.
+
+
+        Example:
+
+        ```yaml
+        RESTClient:
+          url: https://api.upcdatabase.org/product/{barcode}
+          barcode: '0111222333446'
+          credentials:
+            apikey: UPC_API_KEY
+          as_dataframe: true
+        ```
+
+    """
+import asyncio
+from abc import ABC
+from typing import List, Dict, Union
+from collections.abc import Callable
+from urllib.parse import urlencode
+from navconfig.logging import logging
+from ..exceptions import DataNotFound, ComponentError
+from .HTTPClient import HTTPClient
+
+
+class RESTClient(HTTPClient):
+    """
+    RESTClient
+
+    Overview
+
+        The RESTClient class is a component for making RESTful queries to URLs. It extends the HTTPClient class and provides
+        functionality to send requests and process responses from REST APIs. It supports creating DataFrames from JSON responses
+        if specified.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _result          |   No     | The result of the REST query, can be a list or dictionary.                                        |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | accept           |   No     | The accepted response type, defaults to "application/json".                                       |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | url              |   Yes    | The URL to send the REST query to.                                                                |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | method           |   No     | The HTTP method to use for the request, defaults to the method specified in the class.            |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+    Return
+
+        The methods in this class manage the execution of RESTful queries and handle the response. It includes functionality to
+        convert JSON responses into DataFrames if specified.
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        """Init Method."""
+        self._result: Union[List, Dict] = None
+        self._data: dict = kwargs.pop("data", {})
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.accept: str = "application/json"  # by default
+
+    async def run(self):
+        self.url = self.build_url(
+            self.url,
+            args=self.arguments,
+            queryparams=urlencode(self.parameters)
+        )
+        try:
+            if self.use_async is True:
+                result, error = await self.async_request(
+                    url=self.url,
+                    method=self.method,
+                    data=self._data
+                )
+            else:
+                result, error = await self.request(
+                    url=self.url,
+                    method=self.method,
+                    data=self._data
+                )
+            if not result:
+                raise DataNotFound(f"Data was not found on: {self.url}")
+            elif error is not None:
+                if isinstance(error, BaseException):
+                    raise error
+                else:
+                    raise ComponentError(f"RESTClient Error: {error}")
+        except Exception as err:
+            logging.exception(err, stack_info=True)
+            raise ComponentError(f"RESTClient Error: {err}") from err
+        # at here, processing Result
+        if self.as_dataframe is True:
+            try:
+                result = await self.create_dataframe(result)
+            except Exception as err:
+                raise ComponentError(f"RESTClient Error: {err}") from err
+        self._result = result
+        return self._result
+
+
+class AbstractREST(RESTClient):
+    """AbstractREST.
+    Abstract Method for RESTful Components.
+    """
+
+    _default_method: str = None
+    base_url: str = None
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        """Init Method."""
+        self._result: Union[List, Dict] = None
+        self.url: str = None
+        self._method: str = kwargs.pop('method', self._default_method)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.accept: str = "application/json"  # by default
+        self._args = self._params
+
+    async def start(self, **kwargs):
+        if not hasattr(self, self._method):
+            raise ComponentError(
+                f"{self.__name__} Error: has no Method {self._method}"
+            )
+        await super(AbstractREST, self).start(**kwargs)
+
+    async def run(self):
+        method = getattr(self, self._method)
+        try:
+            await method()
+        except Exception as err:
+            logging.exception(err, stack_info=True)
+            raise ComponentError(
+                f"{self.__name__}: Error calling Method {self._method}: {err}"
+            ) from err
+        self.url = self.build_url(
+            self.url,
+            args=self.arguments,
+            queryparams=urlencode(self.parameters)
+        )
+        try:
+            if self.use_async is True:
+                result, error = await self.async_request(
+                    url=self.url,
+                    method=self.method,
+                    data=self._data
+                )
+            else:
+                result, error = await self.request(
+                    url=self.url,
+                    method=self.method,
+                    data=self._data
+                )
+            if self._debug:
+                print(result)
+            if not result:
+                raise DataNotFound(
+                    f"Data was not found on: {self.url}"
+                )
+            elif error is not None:
+                if isinstance(error, BaseException):
+                    raise error
+                else:
+                    raise ComponentError(f"HTTPClient Error: {error}")
+            # at here, processing Result
+            if self.as_dataframe is True:
+                try:
+                    result = await self.create_dataframe(result)
+                    if self._debug is True:
+                        print(result)
+                        print("::: Printing Column Information === ")
+                        for column, t in result.dtypes.items():
+                            print(column, "->", t, "->", result[column].iloc[0])
+                except Exception as err:
+                    raise ComponentError(f"HTTPClient Error: {err}") from err
+            self._result = result
+            return self._result
+        except Exception as err:
+            logging.exception(err, stack_info=True)
+            raise ComponentError(f"HTTPClient Error: {err}") from err