flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/interfaces/dataframes/dt.py

@@ -0,0 +1,69 @@
+from typing import Union, Any, ParamSpec
+from io import BytesIO
+import orjson
+import pyarrow as pa
+import datatable as dt
+import pandas as pd
+from .abstract import BaseDataframe
+from ...exceptions import ComponentError, DataNotFound
+
+
+P = ParamSpec("P")
+
+
+def is_empty(obj):
+    """check_empty.
+    Check if a basic object, a Polars DataFrame, an Apache Arrow Table,
+    or a Python DataTable Frame is empty or not.
+    """
+    if isinstance(obj, (pa.Table, dt.Frame)):
+        return obj.nrows == 0
+    else:
+        return bool(not obj)
+
+
+class DtDataframe(BaseDataframe):
+    """DtDataframe.
+
+    Converts any result into a Datatable DataFrame.
+    """
+
+    async def create_dataframe(
+        self, result: Union[dict, bytes, Any], *args: P.args, **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts any result into a Datatable DataFrame.
+
+        :param result: The result data to be converted into a Datatable DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        if is_empty(result):
+            raise DataNotFound("Frame: No Data was Found.")
+        try:
+            if isinstance(result, str):
+                try:
+                    result = orjson.loads(result)
+                except Exception:
+                    pass
+            if isinstance(result, (list, dict)):
+                df = dt.Frame(result, **kwargs)
+            elif isinstance(result, bytes) or isinstance(result, BytesIO):
+                # Reset the pointer to the start of the stream
+                result.seek(0)
+                # Assuming bytes is a CSV format, adjust as needed
+                df = dt.Frame(pd.read_csv(result))
+            else:
+                raise ValueError("Unsupported data type for DataTable Frame creation")
+
+            columns = list(df.names)
+            numrows = df.nrows
+            numcols = df.ncols
+            try:
+                self._variables["_numRows_"] = numrows
+                self.add_metric("NUM_ROWS", numrows)
+                self.add_metric("NUM_COLS", numcols)
+            except Exception:
+                pass
+            return df
+        except Exception as err:
+            raise ComponentError(f"Error Creating Frame: {err!s}")

flowtask/interfaces/dataframes/pandas.py

@@ -0,0 +1,167 @@
+from typing import Union, Any, ParamSpec, Callable
+import asyncio
+import orjson
+import pandas as pd
+from .abstract import BaseDataframe
+from ...exceptions import ComponentError, DataNotFound
+
+P = ParamSpec("P")
+
+
+def is_empty(obj):
+    """check_empty.
+    Check if a basic object is empty or not.
+    """
+    if isinstance(obj, pd.DataFrame):
+        return True if obj.empty else False
+    else:
+        return bool(not obj)
+
+
+class PandasDataframe(BaseDataframe):
+    """PandasDataframe.
+
+    Converts any result into a Pandas DataFrame.
+    """
+    chunk_size: int = 100
+    task_parts: int = 10
+
+    async def create_dataframe(
+        self,
+        result: Union[dict, bytes, Any],
+        *args: P.args,
+        **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts any result into a Pandas DataFrame.
+
+        :param result: The result data to be converted into a Pandas DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        if is_empty(result):
+            raise DataNotFound("DataFrame: No Data was Found.")
+        try:
+            if isinstance(result, str):
+                try:
+                    result = orjson.loads(result)
+                except Exception:
+                    pass
+            if isinstance(result, dict):
+                result = [result]
+            df = pd.DataFrame(result, **kwargs)
+            # Attempt to infer better dtypes for object columns.
+            df.infer_objects()
+            columns = list(df.columns)
+            if hasattr(self, "infer_types"):
+                df = df.convert_dtypes(convert_string=self.to_string)
+            if hasattr(self, "infer_types"):
+                df = df.convert_dtypes()
+            if hasattr(self, "drop_empty"):
+                df.dropna(axis=1, how="all", inplace=True)
+                df.dropna(axis=0, how="all", inplace=True)
+            if hasattr(self, "dropna"):
+                df.dropna(subset=self.dropna, how="all", inplace=True)
+            numrows = len(df.index)
+            try:
+                self._variables["_numRows_"] = numrows
+                self.add_metric("NUM_ROWS", numrows)
+                self.add_metric("NUM_COLS", len(columns))
+            except Exception:
+                pass
+            return df
+        except Exception as err:
+            raise ComponentError(
+                f"Error Creating Dataframe: {err!s}"
+            )
+
+    async def from_csv(
+        self, result: str, *args: P.args, **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts a Comma-Separated CSV into a Pandas DataFrame.
+
+        :param result: The result data to be converted into a Pandas DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        if is_empty(result):
+            raise DataNotFound("DataFrame: No Data was Found.")
+        try:
+            df = pd.read_csv(result, encoding="utf-8", **kwargs)
+            # Attempt to infer better dtypes for object columns.
+            df.infer_objects()
+            columns = list(df.columns)
+            if hasattr(self, "infer_types"):
+                df = df.convert_dtypes(convert_string=self.to_string)
+            if hasattr(self, "infer_types"):
+                df = df.convert_dtypes()
+            if hasattr(self, "drop_empty"):
+                df.dropna(axis=1, how="all", inplace=True)
+                df.dropna(axis=0, how="all", inplace=True)
+            if hasattr(self, "dropna"):
+                df.dropna(subset=self.dropna, how="all", inplace=True)
+            numrows = len(df.index)
+            try:
+                self._variables["_numRows_"] = numrows
+                self.add_metric("NUM_ROWS", numrows)
+                self.add_metric("NUM_COLS", len(columns))
+            except Exception:
+                pass
+            return df
+        except Exception as err:
+            raise ComponentError(f"Error Creating Dataframe: {err!s}")
+
+    def column_exists(self, column: str):
+        """Returns True if the column exists in the DataFrame."""
+        if column not in self.data.columns:
+            self._logger.warning(
+                f"Column {column} does not exist in the dataframe"
+            )
+            self.data[column] = None
+            return False
+        return True
+
+    def _create_tasks(self, dataframe: pd.DataFrame, func: Union[str, Callable], **kwargs) -> list:
+        """
+        Create tasks for processing the DataFrame.
+
+        :param dataframe: The DataFrame to process.
+        :param
+        func: The function to apply to each row.
+        :return: A list of tasks.
+        """
+        if isinstance(func, str):
+            func = getattr(self, func)
+            if not callable(func):
+                raise ValueError(f"Function {func} is not callable.")
+        if not isinstance(dataframe, pd.DataFrame):
+            raise ValueError(f"Dataframe {dataframe} is not a DataFrame.")
+        if dataframe.empty:
+            raise ValueError(f"Dataframe {dataframe} is empty.")
+        tasks = []
+        for idx, row in dataframe.iterrows():
+            task = func(row, idx, **kwargs)
+            if asyncio.iscoroutinefunction(task):
+                tasks.append(task)
+            else:
+                tasks.append(asyncio.create_task(task))
+        return tasks
+
+    def split_parts(self, task_list, num_parts: int = 5) -> list:
+        part_size = len(task_list) // num_parts
+        remainder = len(task_list) % num_parts
+        parts = []
+        start = 0
+        for i in range(num_parts):
+            # Distribute the remainder across the first `remainder` parts
+            end = start + part_size + (1 if i < remainder else 0)
+            parts.append(task_list[start:end])
+            start = end
+        return parts
+
+    async def _processing_tasks(self, tasks: list) -> pd.DataFrame:
+        """Process tasks concurrently."""
+        results = []
+        for chunk in self.split_parts(tasks, self.task_parts):
+            result = await asyncio.gather(*chunk, return_exceptions=False)
+            results.extend(result)
+        return results

flowtask/interfaces/dataframes/polars.py

@@ -0,0 +1,60 @@
+from typing import Union, Any, ParamSpec
+import orjson
+import pandas as pd
+import polars as pl
+from .abstract import BaseDataframe
+from ...exceptions import ComponentError, DataNotFound
+
+
+P = ParamSpec("P")
+
+
+def is_empty(obj):
+    """check_empty.
+    Check if a basic object or a DataFrame (Pandas or Polars) is empty or not.
+    """
+    if isinstance(obj, pd.DataFrame) or isinstance(obj, pl.DataFrame):
+        return obj.shape[0] == 0
+    else:
+        return bool(not obj)
+
+
+class PolarsDataframe(BaseDataframe):
+    """PolarsDataframe.
+
+    Converts any result into a Polars DataFrame.
+    """
+
+    async def create_dataframe(
+        self, result: Union[dict, bytes, Any], *args: P.args, **kwargs: P.kwargs
+    ) -> Any:
+        """
+        Converts any result into a Polars DataFrame.
+
+        :param result: The result data to be converted into a Polars DataFrame.
+        :return: A DataFrame containing the result data.
+        """
+        if is_empty(result):
+            raise DataNotFound("DataFrame: No Data was Found.")
+        try:
+            if isinstance(result, str):
+                try:
+                    result = orjson.loads(result)
+                except Exception:
+                    pass
+            df = pl.DataFrame(result, **kwargs)
+            columns = list(df.columns)
+            if hasattr(self, "drop_empty"):
+                df = df.drop_nulls(how="all", subset=df.columns)
+            if hasattr(self, "dropna"):
+                df = df.drop_nulls(how="all", subset=self.dropna)
+            numrows = df.height
+            try:
+                self._variables["_numRows_"] = numrows
+                self.add_metric("NUM_ROWS", numrows)
+                self.add_metric("NUM_COLS", df.width)
+            except Exception:
+                pass
+            return df
+        except Exception as err:
+            raise ComponentError(f"Error Creating Dataframe: {err!s}")

flowtask/interfaces/db.py

@@ -0,0 +1,263 @@
+import asyncio
+from typing import Optional
+from collections.abc import Callable
+from navconfig.logging import logging
+from querysource.conf import (
+    # postgres main database:
+    # postgres read-only
+    PG_DRIVER,
+    PG_HOST,
+    PG_USER,
+    PG_PWD,
+    PG_DATABASE,
+    PG_PORT,
+    # rethinkdb
+    RT_DRIVER,
+    RT_HOST,
+    RT_PORT,
+    RT_USER,
+    RT_PASSWORD,
+    RT_DATABASE,
+    # SQL Server
+    MSSQL_DRIVER,
+    MSSQL_HOST,
+    MSSQL_PORT,
+    MSSQL_USER,
+    MSSQL_PWD,
+    MSSQL_DATABASE,
+    # MySQL Server
+    MYSQL_DRIVER,
+    MYSQL_HOST,
+    MYSQL_PORT,
+    MYSQL_USER,
+    MYSQL_PWD,
+    MYSQL_DATABASE,
+    # influxdb
+    INFLUX_DRIVER,
+    INFLUX_HOST,
+    INFLUX_PORT,
+    INFLUX_ORG,
+    INFLUX_TOKEN,
+    # INFLUX_USER,
+    # INFLUX_PWD,
+    INFLUX_DATABASE,
+    # cassandra
+    CASSANDRA_DRIVER,
+    CASSANDRA_HOST,
+    CASSANDRA_PORT,
+    CASSANDRA_USER,
+    CASSANDRA_PWD,
+    CASSANDRA_DATABASE,
+)
+from asyncdb import AsyncDB
+from asyncdb.drivers.base import BaseDriver
+from asyncdb.exceptions import ProviderError
+from ..conf import default_dsn
+from ..exceptions import ComponentError
+from .credentials import CredentialsInterface
+
+
+class DBInterface(CredentialsInterface):
+    """DBInterface.
+
+    Abstract Interface for Database-based connectors.
+    """
+    _credentials = {
+        "host": str,
+        "port": int,
+        "user": str,
+        "password": str,
+        "database": str,
+    }
+
+    def __init__(
+        self,
+        *args,
+        driver: Optional[str] = None,
+        **kwargs
+    ) -> None:
+        self.driver: str = driver
+        self.datasource: str = None
+        self._connection: Callable = None
+        super().__init__(*args, **kwargs)
+
+    async def connection(
+        self,
+        driver: str = "pg",
+        credentials: Optional[dict] = None,
+        event_loop: Optional[asyncio.AbstractEventLoop] = None,
+        **kwargs,
+    ) -> BaseDriver:
+        if not event_loop:
+            event_loop = asyncio.get_event_loop()
+        if driver == "pg":  # default driver:
+            args = {
+                "server_settings": {
+                    "client_min_messages": "notice",
+                    "max_parallel_workers": "24",
+                    "statement_timeout": "36000000",
+                }
+            }
+        else:
+            args = kwargs
+        try:
+            self._connection = AsyncDB(
+                driver, params=credentials, loop=event_loop, **args
+            )
+        except ProviderError as e:
+            raise ComponentError(f"DbClient: Error creating connection: {e}") from e
+        except Exception as e:
+            raise ComponentError(f"DbClient: unknown DB error: {e}") from e
+        if self._connection:
+            return self._connection
+        else:
+            raise ComponentError(f"DbClient: Unable to connect to {driver}")
+
+    def pg_connection(self, event_loop: asyncio.AbstractEventLoop = None) -> BaseDriver:
+        pgargs: dict = {
+            "server_settings": {
+                "application_name": "Flowtask",
+                "client_min_messages": "notice",
+                "max_parallel_workers": "48",
+                "jit": "off",
+                "statement_timeout": "3600000",
+                "effective_cache_size": "2147483647",
+            },
+        }
+        if not event_loop:
+            try:
+                event_loop = asyncio.get_running_loop()
+            except RuntimeError:
+                event_loop = asyncio.get_event_loop()
+        return AsyncDB("pg", dsn=default_dsn, loop=event_loop, **pgargs)
+
+    async def get_driver(self, driver: str, conn: BaseDriver) -> BaseDriver:
+        # TODO: migration to Model
+        result = None
+        query = (
+            "SELECT driver, params, credentials FROM public.datasources where name = '{}'"
+        )
+        try:
+            result, error = await conn.queryrow(query.format(driver))
+            if error:
+                raise ComponentError(
+                    f"DbClient: Error on Getting Datasource: {error!s}"
+                )
+            if result:
+                try:
+                    self.driver = result["driver"]
+                except KeyError as ex:
+                    raise RuntimeError(
+                        f"DbClient Error: there is no *Driver* column on datasource {driver}"
+                    ) from ex
+                try:
+                    params = result["params"]
+                    logging.debug(
+                        f"DB: connection params: {params}"
+                    )
+                    self.credentials = {**dict(params), **dict(result["credentials"])}
+                except (TypeError, ValueError, KeyError) as ex:
+                    raise RuntimeError(
+                        f"DbClient Error: wrong or missing credentials on Datasource {driver}"
+                    ) from ex
+        except Exception as e:
+            logging.exception(f"DB Error: {e}", stack_info=True)
+            raise
+        if not result:
+            # getting the default for any kind of database connection:
+            # based on provider (rethink, mssql, etc) get default connection
+            if driver == "sqlserver":
+                self.driver = MSSQL_DRIVER
+                self.credentials = {
+                    "host": MSSQL_HOST,
+                    "port": MSSQL_PORT,
+                    "database": MSSQL_DATABASE,
+                    "user": MSSQL_USER,
+                    "password": MSSQL_PWD,
+                }
+            elif driver == "cassandra":
+                self.driver = CASSANDRA_DRIVER
+                self.credentials = {
+                    "host": CASSANDRA_HOST,
+                    "port": CASSANDRA_PORT,
+                    "database": CASSANDRA_DATABASE,
+                    "user": CASSANDRA_USER,
+                    "password": CASSANDRA_PWD,
+                }
+            elif driver in ("influx", "influxdb"):
+                self.driver = INFLUX_DRIVER
+                self.credentials = {
+                    "host": INFLUX_HOST,
+                    "port": INFLUX_PORT,
+                    "database": INFLUX_DATABASE,
+                    "org": INFLUX_ORG,
+                    "token": INFLUX_TOKEN,
+                }
+            elif driver in ("rethink", "rethinkdb"):
+                self.driver = RT_DRIVER
+                self.credentials = {
+                    "host": RT_HOST,
+                    "port": RT_PORT,
+                    "database": RT_DATABASE,
+                    "user": RT_USER,
+                    "password": RT_PASSWORD,
+                }
+            elif driver in ("postgres", "postgresql"):
+                self.driver = PG_DRIVER
+                self.credentials = {
+                    "host": PG_HOST,
+                    "port": PG_PORT,
+                    "database": PG_DATABASE,
+                    "user": PG_USER,
+                    "password": PG_PWD,
+                }
+            elif driver == "mysql":
+                self.driver = MYSQL_DRIVER
+                self.credentials = {
+                    "host": MYSQL_HOST,
+                    "port": MYSQL_PORT,
+                    "database": MYSQL_DATABASE,
+                    "user": MYSQL_USER,
+                    "password": MYSQL_PWD,
+                }
+            else:
+                raise ComponentError(f"Unknown Database Driver {driver}")
+
+    async def get_credentials(self) -> BaseDriver:
+        if self.credentials:
+            return True  # credentials are passed to Component
+        if self.datasource is not None:
+            driver = self.datasource
+        else:
+            driver = "pg"
+        if driver in ("db", "pg"):
+            # default credentials
+            self.driver = PG_DRIVER
+            self.credentials = {
+                "user": PG_USER,
+                "password": PG_PWD,
+                "host": PG_HOST,
+                "port": int(PG_PORT),
+                "database": PG_DATABASE,
+            }
+            return True
+        else:
+            # getting from "datasources" table:
+            db = self.pg_connection()
+            try:
+                async with await db.connection() as conn:
+                    await self.get_driver(driver, conn)
+                    if not self.credentials:
+                        raise RuntimeError(
+                            f"DB Error: wrong or missing credentials: {driver}"
+                        )
+            except Exception as e:
+                logging.exception(f"DB Error: {e}")
+                raise
+
+    async def start(self, **kwargs):
+        # first: getting credential from datasource or creds dictionary.
+        # second: processing credentials (extracting value replacements from environment)
+        await self.get_credentials()
+        self.processing_credentials()
+        await super(DBInterface, self).start(**kwargs)

flowtask/interfaces/env.py

@@ -0,0 +1,46 @@
+from abc import ABC
+import os
+from navconfig import config
+
+
+class EnvSupport(ABC):
+    """EnvSupport.
+
+    Support for Environment Variables
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._environment = config
+        # Call super only if there’s a next class in the MRO
+        try:
+            super().__init__(*args, **kwargs)
+        except TypeError:
+            super().__init__()
+
+    def get_env_value(self, key, default: str = None, expected_type: object = None):
+        """
+        Retrieves a value from the environment variables or the configuration.
+
+        :param key: The key for the environment variable.
+        :param default: The default value to return if the key is not found.
+        :param expected_type: the data type to be expected.
+        :return: The value of the environment variable or the default value.
+        """
+        if key is None:
+            return default
+        if expected_type is not None:
+            if expected_type in (int, float):
+                return self._environment.getint(key, default)
+            elif expected_type == bool:
+                return self._environment.getboolean(key, default)
+            else:
+                return self._environment.get(key, default)
+        if val := os.getenv(str(key), default):
+            return val
+        if val := self._environment.get(key, default):
+            return val
+        else:
+            if hasattr(self, "masks") and hasattr(self, "_mask"):
+                if key in self._mask.keys():
+                    return self._mask[key]
+            return key