flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/FilterIf.py

@@ -0,0 +1,284 @@
+import asyncio
+from collections.abc import Callable
+import re
+import pandas as pd
+import numpy as np
+from querysource.types.dt import filters as qsffunctions
+from querysource.queries.multi.operators.filter.flt import (
+    create_filter,
+    valid_operators
+)
+from .FilterRows import functions as dffunctions
+# create_filter
+from ..exceptions import (
+    ConfigError,
+    ComponentError,
+    DataNotFound
+)
+from .flow import FlowComponent
+from . import getComponent
+
+
+class FilterIf(FlowComponent):
+    """
+    FilterIf.
+
+        Overview
+
+            The FilterIf is a component that applies specified filters to a Pandas DataFrame.
+            if the condition is met, the row is kept, otherwise it is discarded.
+            at result set (if any) will be executed a subset of components.
+
+        .. table:: Properties
+        :widths: auto
+
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | Name         | Required | Summary                                                                   |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | operator     |   Yes    | Logical operator (e.g., `and`, `or`) used to combine filter conditions.   |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | conditions   |   Yes    | List of conditions with columns, values, and expressions for filtering.   |
+            |              |          | Format: `{ "column": <col_name>, "value": <val>, "expression": <expr> }`  |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | filter       |   Yes    | List of conditions with columns, values, and expressions for filtering.   |
+            |              |          | Format: `{ "column": <col_name>, "value": <val>, "expression": <expr> }`  |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | true_condition|  Yes    | List of components to execute if the condition is met.                   |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+            | false_condition| Yes    | List of components to execute if the condition is not met.               |
+            +--------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a filtered Pandas DataFrame based on the provided conditions.
+            The component tracks metrics
+            such as the initial and filtered row counts, and optionally limits the returned columns if specified.
+            Additional debugging information can be outputted based on configuration.
+
+
+        Example:
+
+    ```yaml
+    - FilterIf:
+        operator: "&"
+        filter:
+            - column: previous_form_id
+            expression: not_null
+        true_condition:
+            - TransformRows:
+                replace_columns: true
+                fields:
+                form_id: previous_form_id
+            - ExecuteSQL:
+                file_sql: delete_previous_form.sql
+                use_template: true
+                use_dataframe: true
+    ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.condition: str = ""
+        self.fields: dict = kwargs.pop('fields', {})
+        self.operator = kwargs.pop('operator', '&')
+        self.filter = kwargs.pop('filter', [])
+        self.true_condition = kwargs.pop('true_condition', [])
+        self.false_condition = kwargs.pop('false_condition', [])
+        self.filter_conditions: dict = {}
+        super(FilterIf, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        # Si lo que llega no es un DataFrame de Pandas se cancela la tarea
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError(
+                "Data Not Found"
+            )
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError(
+                "Incompatible Pandas Dataframe"
+            )
+        return True
+
+    async def close(self):
+        pass
+
+    def _filter_conditions(self, df: pd.DataFrame) -> pd.DataFrame:
+        it = df.copy()
+        for ft, args in self.filter_conditions.items():
+            self._applied.append(f"Filter: {ft!s} args: {args}")
+            try:
+                try:
+                    func = getattr(qsffunctions, ft)
+                except AttributeError:
+                    try:
+                        func = getattr(dffunctions, ft)
+                    except AttributeError:
+                        func = globals()[ft]
+                if callable(func):
+                    it = func(it, **args)
+            except Exception as err:
+                print(f"Error on {ft}: {err}")
+        df = it
+        if df is None or df.empty:
+            raise DataNotFound(
+                "No Data was Found after Filtering."
+            )
+        return df
+
+    def _filter_fields(self, df: pd.DataFrame) -> pd.DataFrame:
+        for column, value in self.fields.items():
+            if column in df.columns:
+                if isinstance(value, list):
+                    for v in value:
+                        df = df[df[column] == v]
+                else:
+                    df = df[df[column] == value]
+        return df
+
+    def _define_step(self, cpobj: FlowComponent, params: dict) -> Callable:
+        params["ENV"] = self._environment
+        if hasattr(self, "_program"):
+            params["_program"] = self._program
+        # params
+        params["params"] = self._params
+        # parameters
+        params["parameters"] = self._parameters
+        # useful to change variables in set var components
+        params["_vars"] = self._vars
+        # variables dictionary
+        params["variables"] = self._variables
+        params["_args"] = self._args
+        # argument list for components (or tasks) that need argument lists
+        params["arguments"] = self._arguments
+        params["debug"] = self._debug
+        params["argparser"] = self._argparser
+        # the current in-memory connector
+        params["memory"] = self._memory
+        try:
+            job = cpobj(
+                job=self,
+                loop=self._loop,
+                # stat=self.stat,
+                **params
+            )
+            return job
+        except TypeError as err:
+            raise ComponentError(
+                f"Component {cpobj} is not callable: {err}"
+            ) from err
+        except AttributeError as err:
+            raise ComponentError(
+                f"Component {cpobj} not found: {err}"
+            ) from err
+        except Exception as err:
+            raise ComponentError(
+                f"Generic Component Error on {cpobj}, error: {err}"
+            ) from err
+
+    async def _execute_components(
+        self,
+        components: list,
+        df: pd.DataFrame
+    ) -> None:
+        """
+        Execute a list of components with the given DataFrame.
+        Args:
+            components (list): List of components to execute.
+            df (pd.DataFrame): DataFrame to pass to the components.
+        """
+        inner_result = df
+        for component in components:
+            try:
+                # component is a dict, split into key and values:
+                component_name = list(component.keys())[0]
+                args = component[component_name]
+                cpobj = getComponent(component_name)
+                # check if component is callable:
+                if not callable(cpobj):
+                    raise ComponentError(
+                        f"Component {component_name} is not callable."
+                    )
+                step = self._define_step(cpobj, args)
+                step.input = inner_result
+                async with step as comp:
+                    try:
+                        result = await comp.run()
+                        if isinstance(result, bool):
+                            result = step.input
+                        if result is None or result.empty:
+                            self._logger.warning(
+                                f"No Data was Found after Executing {component_name}."
+                            )
+                        self._logger.notice(
+                            f"Component {component_name} executed successfully."
+                        )
+                        inner_result = result
+                    except Exception as e:
+                        self._logger.error(
+                            f"Error executing component {component_name}: {e}"
+                        )
+            except ComponentError as exc:
+                raise ComponentError(
+                    f"Component {component_name} not found: {exc}"
+                ) from exc
+
+    async def run(self):
+        self.add_metric("STARTED_ROWS", len(self.data.index))
+        if not self.filter:
+            raise ConfigError(
+                "No Filter Conditions were Found."
+            )
+        if not self.operator:
+            self.operator = '&'
+        df = self.data.copy()
+        # iterate over all filtering conditions:
+        df = self._filter_conditions(df)
+        # Applying filter expressions by Column:
+        if self.fields:
+            df = self._filter_fields()
+        if self.filter:
+            conditions = create_filter(self.filter, df)
+            # Joining all conditions
+            self.condition = f" {self.operator} ".join(conditions)
+            self._logger.notice(
+                f"Filter conditions >> {self.condition}"
+            )
+            df = df.loc[
+                eval(self.condition)
+            ]  # pylint: disable=W0123
+        if df is None or df.empty:
+            self._logger.warning(
+                "No Data was Found after Filtering."
+            )
+            self._result = self.data
+            return self._result
+        # if the condition is met, execute true_condition
+        if self.true_condition:
+            await self._execute_components(
+                self.true_condition,
+                df
+            )
+        self._result = df
+        self.add_metric(
+            "FILTERED_ROWS", len(self._result.index)
+        )
+        if hasattr(self, "columns"):
+            # returning only a subset of data
+            self._result = self._result[self.columns]
+        if self._debug is True:
+            print("::: Printing Column Information === ")
+            for column, t in self._result.dtypes.items():
+                print(column, "->", t, "->", self._result[column].iloc[0])
+        self.add_metric(
+            "FILTERED_COLS", len(self._result.columns)
+        )
+        return self._result

flowtask/components/FilterRows/FilterRows.py

@@ -0,0 +1,200 @@
+import asyncio
+from collections.abc import Callable
+
+# logging system
+import numpy as np
+import pandas
+from querysource.types.dt import filters as qsffunctions
+from . import functions as dffunctions
+from ..flow import FlowComponent
+from ...utils.functions import check_empty
+from ...exceptions import ComponentError, DataNotFound
+
+
+class FilterRows(FlowComponent):
+    """
+    FilterRows
+
+    Overview
+
+        The FilterRows class is a component for removing or cleaning rows in a Pandas DataFrame based on specified criteria.
+        It supports various cleaning and filtering operations and allows for the saving of rejected rows to a file.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | fields           |   Yes    | A dictionary defining the fields and corresponding filtering conditions to be applied.           |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | filter_conditions|   Yes    | A dictionary defining the filter conditions for transformations.                                 |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _applied         |   No     | A list to store the applied filters.                                                             |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | multi            |   No     | A flag indicating if multiple DataFrame transformations are supported, defaults to False.        |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+    Return
+
+        The methods in this class manage the filtering of rows in a Pandas DataFrame, including initialization, execution,
+        and result handling.
+
+    
+
+        Example:
+
+        ```yaml
+        FilterRows:
+          filter_conditions:
+            clean_empty:
+              columns:
+              - updated
+            drop_columns:
+              columns:
+              - legal_street_address_1
+              - legal_street_address_2
+              - work_location_address_1
+              - work_location_address_2
+              - birth_date
+            suppress:
+              columns:
+              - payroll_id
+              - reports_to_payroll_id
+              pattern: (\.0)
+          drop_empty: true
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.fields: dict = {}
+        self.filter_conditions: dict = {}
+        self._applied: list = []
+        self.multi = bool(kwargs.pop("multi", False))
+        if self.multi:
+            self.fields = {}
+        if self.multi is False:
+            if "fields" in kwargs:
+                self.fields = kwargs.pop('fields', {})
+        else:
+            self.fields = {}
+        super(FilterRows, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe."""
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("a Previous Component was not found.")
+        if check_empty(self.data):
+            raise DataNotFound("No data was found")
+
+    async def run(self):
+        if self.data is None:
+            return False
+        if isinstance(self.data, pandas.DataFrame):
+            # add first metrics
+            self.add_metric("started_rows", self.data.shape[0])
+            self.add_metric("started_columns", self.data.shape[1])
+
+            # start filtering
+            if hasattr(self, "clean_strings"):
+                u = self.data.select_dtypes(include=["object", "string"])
+                self.data[u.columns] = self.data[u.columns].fillna("")
+            if hasattr(self, "clean_numbers"):
+                u = self.data.select_dtypes(include=["Int64"])
+                # self.data[u.columns] = self.data[u.columns].fillna('')
+                self.data[u.columns] = self.data[u.columns].replace(
+                    ["nan", np.nan], 0, regex=True
+                )
+                u = self.data.select_dtypes(include=["float64"])
+                self.data[u.columns] = self.data[u.columns].replace(
+                    ["nan", np.nan], 0, regex=True
+                )
+            if hasattr(self, "clean_dates"):
+                u = self.data.select_dtypes(include=["datetime64[ns]"])
+                self.data[u.columns] = self.data[u.columns].replace({np.nan: None})
+                # df[u.columns] = df[u.columns].astype('datetime64[ns]')
+            if hasattr(self, "drop_empty"):
+                # First filter out those rows which
+                # does not contain any data
+                self.data.dropna(how="all")
+                # removing empty cols
+                self.data.is_copy = None
+                self.data.dropna(axis=1, how="all")
+                self.data.dropna(axis=0, how="all")
+            if hasattr(self, "dropna"):
+                self.data.dropna(subset=self.dropna, how="all")
+            # iterate over all filtering conditions:
+            df = self.data
+            it = df.copy()
+            for ft, args in self.filter_conditions.items():
+                self._applied.append(f"Filter: {ft!s} args: {args}")
+                # TODO: create an expression builder
+                # condition = dataframe[(dataframe[column].empty) & (dataframe[column]=='')].index
+                # check if is a function
+                try:
+                    try:
+                        func = getattr(dffunctions, ft)
+                    except AttributeError:
+                        func = getattr(qsffunctions, ft)
+                    except AttributeError:
+                        func = globals()[ft]
+                    if callable(func):
+                        it = func(it, **args)
+                except Exception as err:
+                    print(f"Error on {ft}: {err}")
+            else:
+                df = it
+            if df is None or df.empty:
+                raise DataNotFound("No Data was Found after Filtering.")
+            self._result = df
+            passed = len(self._result.index)
+            rejected = len(self.data.index) - len(self._result.index)
+            # avoid threat the Dataframe as a Copy
+            self._result.is_copy = None
+            self.add_metric("ended_rows", df.shape[0])
+            self.add_metric("ended_columns", df.shape[1])
+            self.add_metric("PASSED", passed)
+            self.add_metric("REJECTED", rejected)
+            self.add_metric("FILTERS", self._applied)
+            self._variables[f"{self.StepName}_PASSED"] = passed
+            self._variables[f"{self.StepName}_REJECTED"] = rejected
+            if hasattr(self, 'save_rejected'):
+                if self.save_rejected:
+                    # Identify the indices of the rows that were removed
+                    removed_indices = set(self.data.index) - set(self._result.index)
+                    # Select these rows from the original DataFrame
+                    rejected = self.data.loc[list(removed_indices)]
+                    filename = self.save_rejected.get("filename", "rejected_rows.csv")
+                    try:
+                        rejected.to_csv(filename, sep="|")
+                    except IOError:
+                        self._logger.warning(
+                            f"Error writing Rejectd File: {filename}"
+                        )
+                    self.add_metric(
+                        "rejected_file", filename
+                    )
+            if self._debug:
+                self._logger.verbose(
+                    f"PASSED: {passed}, REJECTED: {rejected}",
+                )
+                print("FILTERED ===")
+                print(df)
+                print("::: Printing Column Information === ")
+                for column, t in df.dtypes.items():
+                    print(column, "->", t, "->", df[column].iloc[0])
+            return self._result
+        else:
+            return self._result
+
+    def close(self):
+        pass

flowtask/components/FilterRows/__init__.py

@@ -0,0 +1,10 @@
+"""
+FilterRows.
+
+Component for filtering, dropping, cleaning Pandas Dataframes.
+"""
+
+
+from .FilterRows import FilterRows
+
+__all__ = ["FilterRows"]

flowtask/components/FilterRows/functions.py

@@ -0,0 +1,4 @@
+"""Functions for FilterRows."""
+import re
+import pandas as pd
+import numpy as np

flowtask/components/GCSToBigQuery.py

@@ -0,0 +1,103 @@
+import asyncio
+from typing import Callable, Tuple, List
+from asyncdb import AsyncDB
+from querysource.datasources.drivers.bigquery import bigquery_default
+from .flow import FlowComponent
+from ..exceptions import ComponentError
+
+
+class GCSToBigQuery(FlowComponent):
+    """
+    GCSToBigQuery.
+
+    Este componente carga un archivo CSV desde un bucket específico de GCS a una tabla de BigQuery.
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.bucket_uri: str = kwargs.pop('bucket_uri', None)       # Recibe el object_uri de CSVToGCS
+        self.table_id: str = kwargs.pop('table_id')
+        self.dataset_id: str = kwargs.pop('dataset_id')
+        self.schema: List[dict] = kwargs.pop('schema', None)
+        self.overwrite: bool = kwargs.pop('overwrite', False)
+        self.delete_gcs: bool = kwargs.pop('delete_gcs', False)
+        self.bq = None  # Instancia de AsyncDB
+        super(GCSToBigQuery, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Inicializa el componente configurando la conexión AsyncDB."""
+        # Obtener bucket_uri del componente anterior si no se proporciona directamente
+        if not self.bucket_uri:
+            self.bucket_uri = self.getTaskVar('object_uri')  # Get object_uri set by CSVToGCS
+
+        # Validar parámetros requeridos
+        if not self.bucket_uri or not self.table_id or not self.dataset_id:
+            raise ComponentError("GCSToBigQuery: 'bucket_uri', 'table_id' y 'dataset_id' son parámetros requeridos.")
+
+        if not bigquery_default:
+            raise ComponentError("GCSToBigQuery: 'bigquery_default' no está configurado correctamente.")
+
+        # Obtener credenciales y parámetros del driver
+        credentials = bigquery_default.get_credentials()
+
+        # Inicializar AsyncDB con el driver de BigQuery
+        try:
+            self.bq = AsyncDB("bigquery", params=credentials)
+            self._logger.info("GCSToBigQuery: Instancia de AsyncDB creada exitosamente.")
+        except Exception as e:
+            raise ComponentError(f"GCSToBigQuery: Error al inicializar AsyncDB: {e}") from e
+
+    async def run(self) -> Tuple[str, str]:
+        """Ejecuta la carga del CSV desde GCS a BigQuery."""
+        if not self.bq:
+            raise ComponentError("GCSToBigQuery: AsyncDB no está inicializado. Asegúrate de ejecutar 'start' antes de 'run'.")
+
+        try:
+            async with await self.bq.connection() as conn:
+                # Truncar la tabla si overwrite=True
+                if self.overwrite:
+                    truncated = await conn.truncate_table(
+                        dataset_id=self.dataset_id,
+                        table_id=self.table_id
+                    )
+                    self._logger.info(f"GCSToBigQuery: Tabla '{self.dataset_id}.{self.table_id}' truncada exitosamente.")
+
+                # Cargar el CSV desde GCS a BigQuery
+                load_result = await conn.read_csv_from_gcs(
+                    bucket_uri=self.bucket_uri,
+                    table_id=self.table_id,
+                    dataset_id=self.dataset_id,
+                    schema=self.schema  # Puede ser None para autodetectar
+                )
+                self._logger.info(f"GCSToBigQuery: {load_result}")
+
+                # Guardar el resultado para el siguiente componente (si es necesario)
+                self.setTaskVar('bigquery_load_result', load_result)
+
+                # Opcionalmente eliminar el objeto de GCS
+                if self.delete_gcs:
+                    await conn.delete_gcs_object(
+                        bucket_uri=self.bucket_uri
+                    )
+                    self._logger.info(f"GCSToBigQuery: Objeto GCS '{self.bucket_uri}' eliminado exitosamente.")
+
+                return self.bucket_uri, "Carga exitosa en BigQuery."
+
+        except ComponentError as ce:
+            raise ce  # Re-lanzar errores específicos de componentes
+        except Exception as e:
+            raise ComponentError(f"GCSToBigQuery: Error durante la carga a BigQuery: {e}") from e
+
+    async def close(self):
+        """Cierra la conexión AsyncDB."""
+        try:
+            if self.bq:
+                await self.bq.close()
+                self._logger.info("GCSToBigQuery: AsyncDB cerrado exitosamente.")
+        except Exception as e:
+            self._logger.error(f"GCSToBigQuery: Error al cerrar AsyncDB: {e}")