flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/tGroup.py

@@ -0,0 +1,236 @@
+import asyncio
+from collections.abc import Callable
+import pandas as pd
+import numpy as np
+from ..exceptions import ComponentError, ConfigError
+from .flow import FlowComponent
+
+
+class tGroup(FlowComponent):
+    """
+    tGroup
+
+        Overview
+
+            The tGroup class is a component for performing a group-by operation on a DataFrame using specified columns.
+            It returns unique combinations of the specified group-by columns, allowing data aggregation and summarization.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | group_by       |   Yes    | List of columns to group by.                                              |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | columns        |   No     | List of columns to retain in the result DataFrame. If None,               |
+            |                |          | all columns in `group_by` are returned.                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | agg            |   No     | List of aggregation functions to apply to the grouped data.               |
+            |                |          | Each aggregation should be a dictionary with the column name,             |
+            |                | aggregation function, and an optional alias.                                         |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+        Returns
+
+            This component returns a DataFrame with unique rows based on the specified `group_by` columns. If `columns`
+            is defined, only those columns are included in the result. The component provides debugging information on
+            column data types if enabled, and any errors during grouping are logged and raised as exceptions.
+
+        Example:
+
+        ```
+        - tGroup:
+            group_by:
+            - store_id
+            - formatted_address
+            - state_code
+            - latitude
+            - longitude
+            - store_name
+            - city
+        ```
+
+        - Aggregation Example:
+        ```
+        - tGroup:
+            group_by:
+            - store_id
+            - formatted_address
+            - state_code
+            - latitude
+            - longitude
+            - store_name
+            - city
+            agg:
+                - store_id: distinct
+                alias: unique_store_ids
+                - latitude: mean
+                alias: avg_latitude
+                - longitude: mean
+                alias: avg_longitude
+                - state_code: count
+        ```
+    """  # noqa
+
+    condition = ""
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._columns: list = kwargs.pop("group_by", None)
+        self.aggregations: list = kwargs.pop("agg", [])
+        if not self._columns:
+            raise ConfigError(
+                "tGroup require a list of Columns for Group By => **group_by**"
+            )
+        if not isinstance(self._columns, list):
+            raise ConfigError("Group By must be a list of columns")
+        if not all(isinstance(col, str) for col in self._columns):
+            raise ConfigError("All group_by columns must be strings")
+        super(tGroup, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        # Si lo que llega no es un DataFrame de Pandas se cancela la tarea
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("Data Not Found")
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError("Incompatible Pandas Dataframe")
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        self._result = None
+        try:
+            hashable_columns = [
+                col for col in self._columns
+                if not self.data[col].apply(lambda x: isinstance(x, list)).any()
+            ]
+            if self.aggregations:
+                agg_dict = {}
+                for agg in self.aggregations:
+                    agg = agg.copy()
+                    # agg is a dictionary as:
+                    # - store_id: distinct
+                    #   alias: unique_store_ids
+                    # get key, value of first element on dictionary:
+                    if not isinstance(agg, dict):
+                        raise ConfigError(
+                            f"Aggregation must be a dict with column and agg function, got {agg} instead"
+                        )
+                    col, func = next(iter(agg.items()))
+                    fn = func.lower()
+                    agg.pop(col)
+                    alias = agg.pop("alias", f"{col}_{fn}")
+                    args = agg
+                    if fn == "count":
+                        agg_dict[alias] = (col, 'count')
+                    elif fn == "sum":
+                        agg_dict[alias] = (col, 'sum', args)
+                    elif fn == "mean":
+                        agg_dict[alias] = (col, 'mean', args)
+                    elif fn == "median":
+                        agg_dict[alias] = (col, 'median', args)
+                    elif fn == "min":
+                        agg_dict[alias] = (col, 'min', args)
+                    elif fn == "max":
+                        agg_dict[alias] = (col, 'max', args)
+                    elif fn == "std":
+                        agg_dict[alias] = (col, 'std', args)
+                    elif fn == "var":
+                        agg_dict[alias] = (col, 'var', args)
+                    elif fn == "first":
+                        agg_dict[alias] = (col, 'first', args)
+                    elif fn == "last":
+                        agg_dict[alias] = (col, 'last', args)
+                    elif fn == "unique":
+                        agg_dict[alias] = (col, 'unique', args)
+                    elif fn == "nunique":
+                        agg_dict[alias] = (col, 'nunique', args)
+                    elif fn == "mode":
+                        agg_dict[alias] = (col, 'mode', args)
+                    elif fn == "quantile":
+                        agg_dict[alias] = (col, 'quantile', args)
+                    elif fn == "skew":
+                        agg_dict[alias] = (col, 'skew', args)
+                    elif fn == "kurt":
+                        agg_dict[alias] = (col, 'kurt', args)
+                    elif fn == "mad":
+                        agg_dict[alias] = (col, 'mad', args)
+                    elif fn == "sem":
+                        agg_dict[alias] = (col, 'sem', args)
+                    elif fn == "pct_change":
+                        agg_dict[alias] = (col, 'pct_change', args)
+                    elif fn == "diff":
+                        agg_dict[alias] = (col, 'diff', args)
+                    elif fn == "cumsum":
+                        agg_dict[alias] = (col, 'cumsum', args)
+                    elif fn == "cumprod":
+                        agg_dict[alias] = (col, 'cumprod', args)
+                    elif fn == "cummax":
+                        agg_dict[alias] = (col, 'cummax', args)
+                    elif fn == "cummin":
+                        agg_dict[alias] = (col, 'cummin', args)
+                    elif fn == "distinct":
+                        agg_dict[alias] = (col, pd.Series.nunique)
+                    elif fn == 'count_nulls':
+                        agg_dict[alias] = (col, lambda x: x.isnull().sum())
+                    elif fn == 'count_not_nulls':
+                        agg_dict[alias] = (col, lambda x: x.notnull().sum())
+                    elif fn == 'count_unique':
+                        agg_dict[alias] = (col, lambda x: x.nunique())
+                    elif fn == 'count_distinct':
+                        agg_dict[alias] = (col, lambda x: x.unique().size)
+                    elif fn == 'count_zeros':
+                        agg_dict[alias] = (col, lambda x: (x == 0).sum())
+                    elif hasattr(np, fn):
+                        agg_dict[col] = (col, fn, args)
+                    elif fn == 'apply':
+                        if 'function' not in agg:
+                            raise ConfigError(
+                                "Function must be specified for apply"
+                            )
+                        func = agg['function']
+                        if callable(func):
+                            agg_dict[alias] = (col, func)
+                        else:
+                            raise ConfigError(
+                                f"Function {func} must be callable"
+                            )
+                    else:
+                        raise ConfigError(f"Unsupported aggregation function: {fn}")
+                # Perform group by with aggregation
+                try:
+                    df = self.data.groupby(hashable_columns).agg(**agg_dict).reset_index()
+                except KeyError as err:
+                    raise ConfigError(
+                        f"Invalid columns for aggregation: {err}"
+                    ) from err
+                except Exception as err:
+                    raise ComponentError(
+                        f"Error during aggregation: {err}"
+                    ) from err
+            else:
+                # Perform group by without aggregation, avoiding unhashable types
+                # Get unique elements
+                df = self.data[hashable_columns].drop_duplicates().reset_index(drop=True)
+        except Exception as err:
+            raise ComponentError(f"Generic Error on Data: error: {err}") from err
+        if hasattr(self, "columns"):
+            # returning only a subset of data
+            df = df[self.columns]
+        if self._debug is True:
+            print("::: Printing Column Information === ")
+            print("Grouped: ", df)
+            for column, t in df.dtypes.items():
+                print(column, "->", t, "->", df[column].iloc[0])
+        self._result = df
+        return self._result

flowtask/components/tJoin.py

@@ -0,0 +1,270 @@
+import asyncio
+from typing import Any, Union
+from collections.abc import Callable
+import pandas
+from pandas import DataFrame
+from asyncdb.exceptions import NoDataFound
+from ..exceptions import ComponentError, DataNotFound
+from .flow import FlowComponent
+
+
+class tJoin(FlowComponent):
+    """
+    tJoin
+
+    Overview
+
+        The tJoin class is a component for joining two Pandas DataFrames based on specified join conditions. It supports various join types
+        (such as left, right, inner, and outer joins) and handles different scenarios like missing data, custom join conditions, and multi-source joins.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | df1              |   Yes    | The left DataFrame to join.                                                                       |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | df2              |   Yes    | The right DataFrame to join.                                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | type             |   No     | "left"    | The type of join to perform. Supported values are "left", "right", "inner",           |
+        |                  |          |           | "outer", and "anti-join". When "anti-join" is used, it returns the difference         |
+        |                  |          |           | of B - A, i.e., all rows present in df1 but not in df2.                               |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | depends          |   Yes    | A list of dependencies defining the sources for the join.                                         |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | operator         |   No     | The logical operator to use for join conditions, defaults to "and".                               |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | fk               |   No     | The foreign key or list of keys to use for joining DataFrames.                                    |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | no_copy          |   No     | A flag indicating if copies of the DataFrames should not be made, defaults to True.               |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | join_with        |   No     | A list of additional keys to use for join conditions.                                             |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+    Return
+
+        The methods in this class manage the joining of two Pandas DataFrames, including initialization, execution, and result handling.
+        It ensures proper handling of temporary columns and provides metrics on the joined rows.
+
+    
+
+        Example:
+
+        ```yaml
+        tJoin:
+          depends:
+          - TransformRows_2
+          - QueryToPandas_3
+          type: left
+          fk:
+          - store_number
+          args:
+            validate: many_to_many
+        ```
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        """Init Method."""
+        self.type: str = "left"
+        self.df1: Union[DataFrame, Any] = None
+        self.df2: Union[DataFrame, Any] = None
+        super(tJoin, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe."""
+        if not hasattr(self, "depends"):
+            raise ComponentError(
+                "Missing Dependency (depends) Attribute for declaring Sources."
+            )
+        if self._multi:
+            try:
+                self.df1 = self.previous[0].output()
+            except IndexError as ex:
+                name = self.depends[0]
+                raise ComponentError(f"Missing LEFT Dataframe: {name}") from ex
+            try:
+                self.df2 = self.previous[1].output()
+            except IndexError as ex:
+                name = self.depends[1]
+                raise ComponentError("Missing RIGHT Dataframe") from ex
+        elif hasattr(self, "left"):
+            # TODO: this not work:
+            # think in a persistent structure to save every component after
+            # execution, to get later
+            # discover the "Left" Table
+            try:
+                _, num = self.left.split("_")
+                left = self.JobTask.getJobByID(int(num) - 1)
+                self.df1 = left["component"].output()
+            except KeyError as ex:
+                raise DataNotFound(f"Failed Left Task name: {self.left}") from ex
+        elif hasattr(self, "right"):
+            # discover the "Left" Table
+            try:
+                _, num = self.right.split("_")
+                right = self.JobTask.getJobByID(int(num) - 1)
+                self.df2 = right["component"].output()
+            except KeyError as ex:
+                raise DataNotFound(f"Failed Right Task name: {self.right}") from ex
+        else:
+            raise DataNotFound("Data Was Not Found for Join", status=404)
+        return True
+
+    def cleanup_temp_rows(self, df: pandas.DataFrame = None) -> None:
+        try:
+            self.df1.drop(['_tmp_key_df1'], axis=1, inplace=True)
+            self.df2.drop(['_tmp_key_df2'], axis=1, inplace=True)
+        except KeyError:
+            pass
+        if df is not None:
+            df.is_copy = None
+            # Remember to drop the temporary columns before finalizing the dataframe
+            df.drop(
+                ['_tmp_key_df1', '_tmp_key_df2'],
+                axis=1,
+                inplace=True
+            )
+
+    async def run(self):
+        args = {}
+        if self.df1 is None:
+            raise DataNotFound("Main data Not Found for Join", status=404)
+        if self.df1.empty:
+            raise DataNotFound("Data Was Not Found on Dataframe 1", status=404)
+        if self.type == "left" and (self.df2 is None or self.df2.empty):
+            self._result = self.df1
+            return True
+        elif self.df2 is None or self.df2.empty:
+            raise DataNotFound("Data Was Not Found on Dataframe 2", status=404)
+        if hasattr(self, "no_copy"):
+            args["copy"] = self.no_copy
+        if not self.type:
+            self.type = "inner"
+            args["left_index"] = True
+        if hasattr(self, "args") and isinstance(self.args, dict):
+            args = {**args, **self.args}
+        if hasattr(self, "operator"):
+            operator = self.operator
+        else:
+            operator = "and"
+            if hasattr(self, "fk"):
+                args["on"] = self.fk
+            else:
+                args["left_index"] = True
+        # making a Join between 2 dataframes
+        # Add a unique identifier to both dataframes before the merge
+        self.df1['_tmp_key_df1'] = range(1, len(self.df1) + 1)
+        self.df2['_tmp_key_df2'] = range(1, len(self.df2) + 1)
+        try:
+            if operator == "and":
+                if self.type == "anti-join":
+                    # Perform a left merge, adding an indicator column to track the merge source
+                    df = pandas.merge(
+                        self.df1,
+                        self.df2,
+                        how="left",
+                        suffixes=("", "_right"),
+                        indicator=True,
+                        **args,
+                    )
+
+                    # Filter the rows to keep only those that appear exclusively in the left DataFrame (df1)
+                    # by selecting rows labeled as 'left_only' in the _merge column, then drop the _merge column
+                    df = df[df['_merge'] == 'left_only'].drop(columns=['_merge'])
+
+                    # Remove any columns that were suffixed with '_right', since we only want columns from df1
+                    df = df.loc[:, ~df.columns.str.endswith('_right')]
+
+                else:
+                    df = pandas.merge(
+                        self.df1,
+                        self.df2,
+                        how=self.type,
+                        suffixes=("_left", "_right"),
+                        **args,
+                    )
+
+            else:
+                if hasattr(self, "fk"):
+                    args["left_on"] = self.fk
+                else:
+                    args["left_index"] = True
+                ndf = self.df1
+                sdf = self.df2.copy()
+                merge = []
+                for key in self.join_with:
+                    d = pandas.merge(
+                        ndf,
+                        sdf,
+                        right_on=key,
+                        how=self.type,
+                        suffixes=("_left", None),
+                        **args,
+                    )
+                    ndf = d[d[key].isnull()]
+                    ndf.drop(
+                        ndf.columns[ndf.columns.str.contains("_left")],
+                        axis=1,
+                        inplace=True,
+                    )
+                    ddf = d[d[key].notnull()]
+                    ddf.drop(
+                        ddf.columns[ddf.columns.str.contains("_left")],
+                        axis=1,
+                        inplace=True,
+                    )
+                    merge.append(ddf)
+                # merge the last (not matched) rows
+                merge.append(ndf)
+                df = pandas.concat(merge, axis=0)
+                df.reset_index(drop=True)
+                df.is_copy = None
+        except (ValueError, KeyError) as err:
+            self.cleanup_temp_rows()
+            raise ComponentError(
+                f"Cannot Join with missing Column: {err!s}"
+            ) from err
+        except Exception as err:
+            self.cleanup_temp_rows(df)
+            raise ComponentError(
+                f"Unknown JOIN error {err!s}"
+            ) from err
+        numrows = len(df.index)
+        if numrows == 0:
+            raise DataNotFound(
+                "Cannot make any JOIN, returns zero coincidences"
+            )
+        self._variables[f"{self.StepName}_NUMROWS"] = numrows
+        print("ON END> ", numrows)
+        self.add_metric("TOTAL_ROWS", numrows)
+        try:
+            # After merge, count matched rows from each dataframe
+            matched_rows_df1 = df['_tmp_key_df1'].nunique()
+            matched_rows_df2 = df['_tmp_key_df2'].nunique()
+            # Log or print the count of matched rows
+            print(f"Matched Rows from df1: {matched_rows_df1}")
+            print(f"Matched Rows from df2: {matched_rows_df2}")
+            _rows = {
+                "df1": matched_rows_df1,
+                "df2": matched_rows_df2,
+            }
+            self.add_metric("JOINED_ROWS", _rows)
+            if self._debug is True:
+                print("::: Printing Column Information === ")
+                for column, t in df.dtypes.items():
+                    print(column, "->", t, "->", df[column].iloc[0])
+        finally:
+            # helping some transformations
+            self.cleanup_temp_rows(df)
+            self._result = df
+        return self._result
+
+    async def close(self):
+        pass

flowtask/components/tMap/__init__.py

@@ -0,0 +1,9 @@
+"""
+tMap.
+
+Making Column Transformations using a Mapping JSON file.
+"""
+
+from .tMap import tMap
+
+__all__ = ["tMap"]

flowtask/components/tMap/functions.py

@@ -0,0 +1,54 @@
+"""
+tMap Transformations functions based on Series.
+"""
+from typing import List
+import pandas as pd
+import numpy as np
+
+
+def to_string(series: pd.Series, remove_nan: bool = False, **kwargs) -> pd.Series:
+    """to_string.
+
+    Converting to string a Pandas column (Series)
+    Args:
+        series (pandas.Series): Column Series to be converted
+        remove_nan (bool, optional): remove Not a Number from Column. Defaults to False.
+
+    Returns:
+        pandas.Series: a New Serie is returned with string values.
+    """
+    series = series.astype("string")
+    if remove_nan is True:
+        series = series.replace(np.nan, "", regex=True)
+    return series
+
+
+def to_integer(series: pd.Series, **kwargs):
+    """
+    Converts a pandas Series to an integer type, handling errors by coercing invalid values to NaN.
+
+    :param series: The pandas Series to be converted.
+    :param kwargs: Additional keyword arguments.
+    :return: The converted pandas Series with integer type.
+    """
+    try:
+        series = pd.to_numeric(series, errors="coerce")
+        series = series.astype("Int64", copy=False)
+    except Exception as err:
+        print(f"Error on to_Integer: {err}")
+    return series
+
+
+def concat(df: pd.DataFrame, columns: List[str], sep: str = " ") -> pd.Series:
+    """
+    Concatenates the values of the specified columns in the given DataFrame.
+
+    :param df: The input DataFrame
+    :param columns: The list of columns to concatenate
+    :param sep: The separator to use between the concatenated values (default is a space)
+    :return: A Series with the concatenated values
+    """
+    combined = df[columns[0]].astype(str)
+    for col in columns[1:]:
+        combined += sep + df[col].astype(str)
+    return combined