flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/tAutoincrement.py

@@ -0,0 +1,117 @@
+import asyncio
+from collections.abc import Callable
+import pandas as pd
+from ..exceptions import ComponentError, DataNotFound, TaskError
+from .flow import FlowComponent
+from ..interfaces import DBSupport
+
+
+class tAutoincrement(DBSupport, FlowComponent):
+    """
+    tAutoincrement
+
+    Overview
+
+    The `tAutoincrement` component is designed to automatically increment values in a specific column of a dataset.
+    This is particularly useful when you need to fill in missing (null) values in a column with unique, sequential
+    integers starting from the maximum value currently present in the column.
+
+    Properties
+
+    .. table:: Properties
+    :widths: auto
+
+    +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+    | Name             | Required | Type      | Description                                                                          |
+    +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+    | datasource       | Yes      | str       | The datasource name (e.g., schema name) where the dataset is located.                |
+    +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+    | dataset          | Yes      | str       | The name of the dataset (e.g., table name) to work on.                               |
+    +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+    | column           | Yes      | str       | The name of the column in which values will be auto-incremented.                     |
+    +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+    Return
+
+        Returns the dataframe with the given column and its auto-incremented sequence.
+
+    
+
+        Example:
+
+        ```yaml
+        tAutoincrement:
+          skipError: skip
+          datasource: pokemon
+          dataset: districts
+          column: district_id
+          description: Auto-increment district_id for new districts
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        """Init Method."""
+        self.datasource: str = None
+        self.dataset: str = None
+        self.column: str = None
+        self.pd_args = kwargs.pop("pd_args", {})
+        super(tAutoincrement, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Start method."""
+        if not self.datasource:
+            raise TaskError("Missing *datasource* parameter.")
+        if not self.dataset:
+            raise TaskError("Missing *dataset* parameter.")
+        if not self.column:
+            raise TaskError("Missing *column* parameter.")
+
+        # Set the DataFrame if provided via kwargs
+        self.data = kwargs.get('data', None)
+        if self.data is None and hasattr(self, 'previous'):
+            self.data = self.previous.output()
+
+        if self.data is None:
+            raise ComponentError(
+                "No input DataFrame provided to tAutoincrement component."
+            )
+        await super().start(**kwargs)
+        self.processing_credentials()
+        return True
+
+    async def run(self):
+        """Run method to fetch the max value and auto-increment."""
+        # Establishing the connection
+        self.connection = self.pg_connection()
+        try:
+            async with await self.connection.connection() as conn:
+                query = f"SELECT MAX({self.column}) as alias_column FROM {self.datasource}.{self.dataset};"
+                result = await conn.fetchval(query)
+                if result is None:
+                    result = 0
+                self._logger.info(f"Executing query: {query} MAX: {result}")
+                # Incrementing and assigning the values to null columns
+                if self.data.empty:
+                    raise DataNotFound("Input DataFrame is empty.")
+
+                mask = self.data[self.column].isnull()
+                self.data.loc[mask, self.column] = range(result + 1, result + 1 + mask.sum())
+                self._result = self.data
+
+        except Exception as ex:
+            raise ComponentError(f"Error in tAutoincrement: {str(ex)}") from ex
+
+        finally:
+            self.connection = None
+
+        return True
+
+    async def close(self):
+        pass

flowtask/components/tConcat.py

@@ -0,0 +1,109 @@
+import asyncio
+from typing import Any
+from collections.abc import Callable
+import pandas
+from ..exceptions import ComponentError, DataNotFound
+from .flow import FlowComponent
+
+
+class tConcat(FlowComponent):
+    """
+    tConcat
+
+        Overview
+
+            The tConcat class is a component for merging (concatenating) two DataFrames along a specified axis.
+            It supports handling multiple DataFrames and configurable options for the concatenation, with metrics
+            tracking for input and output row counts.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | df1            |   Yes    | The first DataFrame to concatenate.                                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | df2            |   Yes    | The second DataFrame to concatenate.                                      |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | args           |   No     | Dictionary of arguments to pass to `pandas.concat`, such as `axis`.       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a concatenated DataFrame based on the specified axis and additional arguments.
+            Metrics are recorded for the row counts of both input DataFrames and the final concatenated DataFrame.
+            If either DataFrame is missing or empty, an error is raised with a descriptive message.
+
+
+        Example:
+
+        ```yaml
+        tConcat:
+          depends:
+          - TransformRows_8
+          - TransformRows_15
+          args:
+            axis: 0
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.df1: Any = None
+        self.df2: Any = None
+        self.type = None
+        super(tConcat, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe.
+        TODO: iterate over all dataframes.
+        """
+        if self._multi:
+            self.df1 = self.previous[0].output()
+            self.df2 = self.previous[1].output()
+        return True
+
+    async def run(self):
+        args = {}
+        if self.df1.empty:
+            raise DataNotFound("Data Was Not Found on Dataframe 1")
+        elif self.df2 is None or self.df2.empty:
+            raise DataNotFound("Data Was Not Found on Dataframe 2")
+        if hasattr(self, "args") and isinstance(self.args, dict):
+            args = {**args, **self.args}
+        if "axis" not in args:
+            args["axis"] = 1
+        # Adding Metrics:
+        _left = len(self.df1.index)
+        self.add_metric("LEFT: ", _left)
+        _right = len(self.df2.index)
+        self.add_metric("RIGHT: ", _right)
+        # Concat two dataframes
+        try:
+            df = pandas.concat([self.df1, self.df2], **args)
+        except Exception as err:
+            raise ComponentError(
+                f"Error Merging Dataframes: {err}"
+            ) from err
+        numrows = len(df.index)
+        if numrows == 0:
+            raise DataNotFound(
+                "Concat: Cannot make any Merge"
+            )
+        self._variables[f"{self.StepName}_NUMROWS"] = numrows
+        self.add_metric("JOINED: ", numrows)
+        df.is_copy = None
+        print(df)
+        self._result = df
+        return self._result
+
+    async def close(self):
+        pass

flowtask/components/tExplode.py

@@ -0,0 +1,119 @@
+import asyncio
+from typing import Any
+from collections.abc import Callable
+import pandas
+from pandas import json_normalize
+from ..exceptions import ComponentError, DataNotFound
+from .flow import FlowComponent
+
+
+class tExplode(FlowComponent):
+    """
+    tExplode
+
+        Overview
+
+            The tExplode class is a component for transforming a DataFrame by converting a column of lists or dictionaries
+            into multiple rows. It supports options for dropping the original column after exploding, and for expanding
+            nested dictionary structures into separate columns.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+-------------------------------------------------------------------------------+
+            | Name           | Required | Summary                                                                                   |
+            +----------------+----------+-----------+-------------------------------------------------------------------------------+
+            | column         |   Yes    | The name of the column to explode into multiple rows.                                     |
+            +----------------+----------+-----------+-------------------------------------------------------------------------------+
+            | drop_original  |   No     | Boolean indicating if the original column should be dropped after exploding.              |
+            +----------------+----------+-----------+-------------------------------------------------------------------------------+
+            | explode_dataset|   No     | Boolean specifying if nested dictionaries in the column should be expanded as new columns.|
+            +----------------+----------+-----------+-------------------------------------------------------------------------------+
+
+        Returns
+
+            This component returns a DataFrame with the specified column exploded into multiple rows. If `explode_dataset` is
+            set to True and the column contains dictionaries, these are expanded into new columns. Metrics on the row count
+            after explosion are recorded, and any errors encountered during processing are logged and raised as exceptions.
+    
+
+        Example:
+
+        ```yaml
+        tExplode:
+          column: reviews
+          drop_original: false
+        ```
+
+    """ # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.data: Any = None
+        # Column to be exploded
+        self.column: str = kwargs.pop("column", None)
+        self.drop_original: bool = kwargs.pop("drop_original", False)
+        # Useful when exploded column is also composed of dictionary, the dictionary
+        # is also exploded as columns
+        self.explode_dataset: bool = kwargs.pop("explode_dataset", True)
+        super(tExplode, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        # Si lo que llega no es un DataFrame de Pandas se cancela la tarea
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("Data Not Found", status=404)
+        if not isinstance(self.data, pandas.DataFrame):
+            raise ComponentError("Incompatible Pandas Dataframe", status=404)
+        return True
+
+    async def run(self):
+        args = {}
+        if self.data.empty:
+            raise DataNotFound("Data Was Not Found on Dataframe 1")
+
+        # Explode the Rows:
+        try:
+            # Step 1: Explode the 'field' column
+            exploded_df = self.data.explode(self.column)
+            # Reset index to ensure it's unique
+            exploded_df = exploded_df.reset_index(drop=True)
+            if self.explode_dataset is True:
+                # Step 2: Normalize the JSON data in 'exploded' col
+                # This will create a new DataFrame where each dictionary key becomes a column
+                data_df = json_normalize(exploded_df[self.column])
+                # Step 3: Concatenate with the original DataFrame
+                # Drop the original column from exploded_df and join with data_df
+                if self.drop_original is True:
+                    exploded_df.drop(self.column, axis=1)
+                df = pandas.concat([exploded_df, data_df], axis=1)
+            else:
+                df = exploded_df
+        except Exception as err:
+            raise ComponentError(f"Error Merging Dataframes: {err}") from err
+        numrows = len(df.index)
+        if numrows == 0:
+            raise DataNotFound("Concat: Cannot make any Explode")
+        if hasattr(self, "index"):
+            df[self.index] = df["id"]
+            df.drop("id", axis="columns", inplace=True)
+        self._variables[f"{self.StepName}_NUMROWS"] = numrows
+        self.add_metric("EXPLODE: ", numrows)
+        df.is_copy = None
+        print(df)
+        self._result = df
+        if self._debug is True:
+            print("::: Printing Column Information === ")
+            for column, t in df.dtypes.items():
+                print(column, "->", t, "->", df[column].iloc[0])
+        return self._result
+
+    async def close(self):
+        pass

flowtask/components/tFilter.py

@@ -0,0 +1,184 @@
+import asyncio
+from collections.abc import Callable
+import re
+import pandas as pd
+import numpy as np
+from querysource.types.dt import filters as qsffunctions
+from querysource.queries.multi.operators.filter.flt import (
+    create_filter,
+    valid_operators
+)
+from .FilterRows import functions as dffunctions
+# create_filter
+from ..exceptions import (
+    ConfigError,
+    ComponentError,
+    DataNotFound
+)
+from .flow import FlowComponent
+
+
+class tFilter(FlowComponent):
+    """
+    tFilter
+
+        Overview
+
+            The tFilter class is a component that applies specified filters to a Pandas DataFrame.
+            It allows filtering rows based on multiple conditions and expressions, enabling targeted
+            data extraction within a task flow.
+
+        .. table:: Properties
+        :widths: auto
+
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | Name         | Required | Summary                                                                   |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | operator     |   Yes    | Logical operator (e.g., `and`, `or`) used to combine filter conditions.   |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | conditions   |   Yes    | List of conditions with columns, values, and expressions for filtering.   |
+            |              |          | Format: `{ "column": <col_name>, "value": <val>, "expression": <expr> }`  |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a filtered Pandas DataFrame based on the provided conditions.
+            The component tracks metrics
+            such as the initial and filtered row counts, and optionally limits the returned columns if specified.
+            Additional debugging information can be outputted based on configuration.
+
+
+        Example:
+
+        ```yaml
+        tFilter:
+          operator: '&'
+          filter:
+          - column: ClientId
+            value:
+            - 11076
+            expression: ==
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.condition: str = ""
+        self.fields: dict = kwargs.pop('fields', {})
+        self.operator = kwargs.pop('operator', '&')
+        self.filter = kwargs.pop('filter', [])
+        self.filter_conditions: dict = {}
+        super(tFilter, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        # Si lo que llega no es un DataFrame de Pandas se cancela la tarea
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("Data Not Found")
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError("Incompatible Pandas Dataframe")
+        return True
+
+    async def close(self):
+        pass
+
+    def _filter_conditions(self, df: pd.DataFrame) -> pd.DataFrame:
+        it = df.copy()
+        for ft, args in self.filter_conditions.items():
+            self._applied.append(f"Filter: {ft!s} args: {args}")
+            try:
+                try:
+                    func = getattr(qsffunctions, ft)
+                except AttributeError:
+                    try:
+                        func = getattr(dffunctions, ft)
+                    except AttributeError:
+                        func = globals()[ft]
+                if callable(func):
+                    it = func(it, **args)
+            except Exception as err:
+                print(f"Error on {ft}: {err}")
+        df = it
+        if df is None or df.empty:
+            raise DataNotFound(
+                "No Data was Found after Filtering."
+            )
+        return df
+
+    def _filter_fields(self, df: pd.DataFrame) -> pd.DataFrame:
+        for column, value in self.fields.items():
+            if column in df.columns:
+                if isinstance(value, list):
+                    for v in value:
+                        df = df[df[column] == v]
+                else:
+                    df = df[df[column] == value]
+        return df
+
+    async def run(self):
+        self.add_metric("STARTED_ROWS", len(self.data.index))
+        df = self.data.copy()
+        # iterate over all filtering conditions:
+        df = self._filter_conditions(df)
+        # Applying filter expressions by Column:
+        if self.fields:
+            df = self._filter_fields()
+        if self.filter:
+            # conditions = self._create_filter()
+            conditions = create_filter(self.filter, df)
+            # Joining all conditions
+            self.condition = f" {self.operator} ".join(conditions)
+            self._logger.notice(
+                f"Filter conditions >> {self.condition}"
+            )
+            df = df.loc[
+                eval(self.condition)
+            ]  # pylint: disable=W0123
+        if df is None or df.empty:
+            raise DataNotFound(
+                "No Data was Found after Filtering."
+            )
+        self._result = df
+        # print(": Filtered : ")
+        # print(self._result)
+        self.add_metric(
+            "FILTERED_ROWS", len(self._result.index)
+        )
+        # Calculate the rejected rows between self.data and df dataframe:
+        self.add_metric(
+            "REJECTED_ROWS", len(self.data.index) - len(self._result.index)
+        )
+        if hasattr(self, "save_rejected"):
+            # Identify the indices of the rows that were removed
+            removed_indices = set(self.data.index) - set(self._result.index)
+            # Select these rows from the original DataFrame
+            rejected = self.data.loc[list(removed_indices)]
+            filename = self.mask_replacement(
+                self.save_rejected.get("filename", "rejected_rows.csv")
+            )
+            try:
+                rejected.to_csv(filename, sep="|")
+            except IOError:
+                self._logger.warning(f"Error writing Rejectd File: {filename}")
+            self.add_metric(
+                "rejected_file", filename
+            )
+        if hasattr(self, "columns"):
+            # returning only a subset of data
+            self._result = self._result[self.columns]
+        if self._debug is True:
+            print("::: Printing Column Information === ")
+            for column, t in self._result.dtypes.items():
+                print(column, "->", t, "->", self._result[column].iloc[0])
+        self.add_metric(
+            "FILTERED_COLS", len(self._result.columns)
+        )
+        return self._result