flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/tOrder.py

@@ -0,0 +1,93 @@
+import asyncio
+from typing import Union
+from collections.abc import Callable
+from ..exceptions import ComponentError, ConfigError
+from .tPandas import tPandas
+
+class tOrder(tPandas):
+    """
+        tOrder
+
+        Overview
+
+        The `tOrder` class is a component designed to order a Pandas DataFrame by a specified column.
+        It allows sorting the DataFrame either in ascending or descending order based on the specified column.
+
+        Properties
+
+        .. table:: Properties
+        :widths: auto
+
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | Name             | Required | Type      | Description                                                                       |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | columns          | Yes      | str       | The name of the column to sort the DataFrame by.                                  |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | ascending        | No       | bool      | Specifies whether to sort the DataFrame in ascending order. Defaults to True.     |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+
+        Return
+           The dataframe ordinated by the column give it in the order_by either ascending or descending.
+
+    
+
+        Example:
+
+        ```yaml
+        tOrder:
+          columns:
+          - district_id
+          ascending: true
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._column: Union[str, list] = kwargs.pop("columns", None)
+        if isinstance(self._column, list):
+            ascending = [True for x in self._column]
+        elif isinstance(self._column, str):
+            ascending = [True]
+            self._column = [self._column]
+        self._ascending: Union[bool, list] = kwargs.pop("ascending", ascending)
+        if not self._column:
+            raise ConfigError(
+                "tOrder requires a column for ordering => **columns**"
+            )
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def _run(self):
+        try:
+            # Check if the specified column exists in the DataFrame
+            columns = self.data.columns
+            for col in self._column:
+                if col not in columns:
+                    self._logger.warning(
+                        f"The column '{self._column}' does not exist in the DataFrame."
+                    )
+                    return self.data  # Return the unsorted DataFrame
+                # Check if the specified column is empty
+                if self.data[self._column].empty:
+                    self._logger.warning(
+                        f"The column '{self._column}' is empty."
+                    )
+                    return self.data
+            # Sort the DataFrame by the specified column
+            df = self.data.sort_values(
+                by=self._column,
+                ascending=self._ascending,
+                **self.pd_args
+            ).reset_index(drop=True)
+            self.add_metric('ORDER_BY', self._column)
+            return df
+        except Exception as err:
+            raise ComponentError(
+                f"Generic Error on Data: error: {err}"
+            ) from err

flowtask/components/tPandas.py

@@ -0,0 +1,94 @@
+import asyncio
+from typing import Any, Union
+from collections.abc import Callable
+import pandas
+from pandas import DataFrame
+from ..exceptions import ComponentError, DataNotFound
+from .flow import FlowComponent
+
+
+class tPandas(FlowComponent):
+    """
+    tPandas
+
+        Overview
+
+            The tPandas class is an abstract interface for performing various data transformations on Pandas DataFrames.
+            It provides foundational methods and structure for components that need to apply transformations, merges, or other
+            DataFrame operations within a task.
+
+            This interface provides methods to initialize, transform, and debug Pandas DataFrame operations.
+            Concrete implementations using `tPandas` can define specific transformations. On execution, metrics
+            for rows and columns are recorded, and any transformation errors or data mismatches are raised as exceptions
+            with detailed error messages for effective debugging.
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.df1: Union[DataFrame, Any] = None
+        self.df2: Union[DataFrame, Any] = None
+        self.type: str = None
+        self.condition: str = ''
+        # Pandas Arguments:
+        self.pd_args = kwargs.pop("pd_args", {})
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data: DataFrame = self.input
+
+            if hasattr(self, "_multi") and self._multi:
+                self.df1: DataFrame = self.previous[0].output()
+                try:
+                    self.df2: DataFrame = self.previous[1].output()
+                except IndexError:
+                    self.df2 = None
+            else:
+                if not isinstance(self.data, DataFrame):
+                    raise ComponentError("Incompatible Pandas Dataframe", status=404)
+        else:
+            raise DataNotFound("Data Not Found", status=404)
+
+        await super().start(**kwargs)
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        self._result = self.data
+        try:
+            df = await self._run()
+            if df.empty:
+                raise DataNotFound(
+                    f"Data not Found over {self.__name__}"
+                )
+            self._result = df
+            self.add_metric("NUM_ROWS", self._result.shape[0])
+            self.add_metric("NUM_COLUMNS", self._result.shape[1])
+            if self._debug:
+                print(f"Debugging: {self.__name__} ===")
+                print(self._result)
+                columns = list(self._result.columns)
+                for column in columns:
+                    t = self._result[column].dtype
+                    print(
+                        column, "->", t, "->", self._result[column].iloc[0]
+                    )
+            return self._result
+        except DataNotFound:
+            raise
+        except (ValueError, KeyError) as err:
+            raise ComponentError(
+                f"{self.__name__} Error: {err!s}"
+            ) from err
+        except Exception as err:
+            raise ComponentError(
+                f"{self.__name__} Exception {err!s}"
+            ) from err

flowtask/components/tPivot.py

@@ -0,0 +1,71 @@
+import asyncio
+from typing import Union
+from collections.abc import Callable
+from ..exceptions import ComponentError, ConfigError
+from .tPandas import tPandas
+
+class tPivot(tPandas):
+    """
+        tPivot
+
+        Overview
+
+        Pivoting a Dataframe to transpose a column into other columns.
+
+        Properties
+
+        .. table:: Properties
+        :widths: auto
+
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | Name             | Required | Type      | Description                                                                       |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | columns          | Yes      | list      | The List of Columns to be Pivoted.                                                |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | index            | No       | list      | List of columns to be preserved, default to all columns less "values"             |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+        | values           | Yes      | str       | Columns that transpose the values for pivoted column(s).                          |
+        +------------------+----------+-----------+-----------------------------------------------------------------------------------+
+
+        Return
+           The dataframe Pivoted by "columns" with values using the list of "values".
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._columns: Union[str, list] = kwargs.pop("columns", None)
+        self._index: list = kwargs.pop("index", [])
+        self._values: list = kwargs.pop("values", [])
+        self._aggfunc: str = kwargs.pop('aggfunc', 'sum')
+        self._sort: list = kwargs.pop('sort_by', [])
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+    async def _run(self):
+        try:
+            df_pivot = self.data.pivot_table(
+                index=self._index,
+                columns=self._columns,
+                values=self._values,
+                aggfunc=self._aggfunc,
+                dropna=False
+            ).reset_index()
+            # Renaming the columns to match the desired output
+            df_pivot.columns.name = None  # Remove the index name
+            if self._sort:
+                df_pivot = df_pivot.sort_values(
+                    by=self._sort,
+                    ascending=True
+                ).reset_index(drop=True)
+            return df_pivot
+        except Exception as err:
+            raise ComponentError(
+                f"Generic Error on Data: error: {err}"
+            ) from err

flowtask/components/tPluckCols.py

@@ -0,0 +1,76 @@
+import asyncio
+from typing import List
+from collections.abc import Callable
+import pandas
+from ..exceptions import ComponentError
+from .tPandas import tPandas
+
+
+class tPluckCols(tPandas):
+    """
+    tPluckCols
+
+        Overview
+
+            The tPluckCols class is a component for selecting a specific subset of columns from a Pandas DataFrame.
+            It provides a streamlined way to filter columns, allowing only specified columns to be retained in the output.
+
+        .. table:: Properties
+        :widths: auto
+
+            +-------------+----------+-----------+---------------------------------------------------------------+
+            | Name        | Required | Summary                                                                   |
+            +-------------+----------+-----------+---------------------------------------------------------------+
+            | columns     |   Yes    | A list of column names to retain in the DataFrame.                        |
+            +-------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a Pandas DataFrame containing only the specified columns listed in `columns`.
+            If no columns are provided, it raises an error. The resulting DataFrame is a copy to ensure
+            any modifications do not affect the original DataFrame.
+    
+
+        Example:
+
+        ```yaml
+        tPluckCols:
+          depends:
+          - TransformRows_11
+          columns:
+          - location_code
+          - ss_market
+          - store_id
+          - store_number
+          - company_id
+          - pt_ft
+          - is_covered
+          - endcap
+          - rev_band
+          - comparison_store
+        ```
+
+    """ #noqa
+
+
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.columns: List = None
+        super(tPluckCols, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe."""
+        await super().start(**kwargs)
+        if not self.columns:
+            raise ComponentError("Error: need to specify a list of *columns*")
+        return True
+
+    async def _run(self):
+        return self.data[self.columns].copy()

flowtask/components/tUnnest.py

@@ -0,0 +1,82 @@
+import asyncio
+from collections.abc import Callable
+from ..exceptions import ComponentError, ConfigError
+from .tPandas import tPandas
+
+
+class tUnnest(tPandas):
+    """
+    tUnnest
+
+        Overview
+
+            The tUnnest class is a component for splitting a column in a DataFrame into multiple rows, based on a specified
+            separator. This component supports options to drop the original column after splitting and to define a new column
+            for the split values.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | source_column  |   Yes    | The name of the column to split into multiple rows.                    |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | destination    |   No     | The name of the column to store the split values. Defaults to source.  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | drop_source    |   No     | Boolean indicating if the original column should be dropped after split.|
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | separator      |   No     | The separator used to split the values. Defaults to ", ".              |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a DataFrame where the specified `source_column` is split into multiple rows based on the
+            `separator`. If `drop_source` is set to True, the original column is removed after the split. Errors related to
+            column splitting are logged and raised as exceptions.
+    
+
+        Example:
+
+        ```yaml
+        tUnnest:
+          source_column: warehouse_store_ids
+          destination: store_id
+          drop_source: true
+        ```
+
+    """ #noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.source_column: str = kwargs.pop('source_column', None)
+        self.destination: str = kwargs.get('destination', None)
+        self.drop_source: bool = kwargs.get('drop_source', False)
+        self.separator: str = kwargs.get('separator', ', ')
+        if not self.source_column:
+            raise ConfigError(
+                "Missing Source_column for making unnest."
+            )
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def _run(self):
+        try:
+            # Split the column into multiple rows
+            df = self.data.assign(
+                **{
+                    self.destination: self.data[self.source_column].str.split(self.separator)
+                }
+            ).explode(self.destination)
+            if self.drop_source is True:
+                # Drop the original column
+                df = df.drop(columns=[self.source_column])
+            return df
+        except Exception as err:
+            raise ComponentError(
+                f"Unknown error {err!s}"
+            ) from err