cloe-nessy 0.2.11__py3-none-any.whl → 0.3.2__py3-none-any.whl

cloe_nessy/pipeline/actions/__init__.py

@@ -12,6 +12,7 @@ from .transform_decode import TransformDecodeAction
 from .transform_distinct import TransformDistinctAction
 from .transform_filter import TransformFilterAction
 from .transform_generic_sql import TransformSqlAction
+from .transform_group_aggregate import TransformGroupAggregate
 from .transform_join import TransformJoinAction
 from .transform_json_normalize import TransformJsonNormalize
 from .transform_rename_columns import TransformRenameColumnsAction
@@ -42,6 +43,7 @@ __all__ = [
     "TransformDecodeAction",
     "TransformDistinctAction",
     "TransformSqlAction",
+    "TransformGroupAggregate",
     "TransformJoinAction",
     "TransformJsonNormalize",
     "TransformRenameColumnsAction",

cloe_nessy/pipeline/actions/transform_deduplication.py

@@ -0,0 +1,125 @@
+import random
+import string
+from typing import Any
+
+import pyspark.sql.functions as F
+import pyspark.sql.types as T
+from pyspark.sql import Window
+
+from ..pipeline_action import PipelineAction
+from ..pipeline_context import PipelineContext
+
+
+class TransformDeduplication(PipelineAction):
+    """Deduplicates the data from the given DataFrame.
+
+    This method deduplicates the data where the key columns are the same
+    and keeps the entry with the highest values in the order_by_columns
+    (can be changed to lowest by setting the parameter descending to false).
+
+    Example:
+    ```yaml
+    Deduplicate Columns:
+        action: TRANSFORM_DEDUPLICATION
+        options:
+            key_columns:
+                - id
+            order_by_columns:
+                - source_file_modification_time
+    ```
+    """
+
+    name: str = "TRANSFORM_DEDUPLICATION"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        key_columns: list[str] | None = None,
+        order_by_columns: list[str] | None = None,
+        descending: bool = True,
+        **_: Any,
+    ) -> PipelineContext:
+        """Deduplicates the data based on key columns and order by columns.
+
+        Args:
+            context: The context in which this Action is executed.
+            key_columns: A list of the key column names. The returned data only keeps one
+                line of data with the same key columns.
+            order_by_columns: A list of order by column names. The returned data keeps the
+                first line of data with the same key columns ordered by these columns.
+            descending: Whether to sort descending or ascending.
+
+        Raises:
+            ValueError: If no key_columns are specified.
+            ValueError: If no order_by_columns are specified.
+            ValueError: If the data from context is None.
+            ValueError: If key_columns and order_by_columns overlap.
+            ValueError: If key_columns or order_by_columns contain Nulls.
+
+        Returns:
+            The context after the execution of this Action, containing the DataFrame with the deduplicated data.
+        """
+        if context.data is None:
+            raise ValueError("Data from the context is required for the operation.")
+        if key_columns is None:
+            raise ValueError("Please provide at least one key column.")
+        if order_by_columns is None:
+            raise ValueError("Please provide at least one order by column.")
+
+        # check if the key_columns and order_by_columns are the same
+        if len(set(key_columns) & set(order_by_columns)) != 0:
+            raise ValueError("The key_columns and order_by_columns cannot contain the same column")
+
+        # check if the key_columns and order_by_columns are not null
+        df_nulls = context.data.filter(F.greatest(*[F.col(c).isNull() for c in key_columns + order_by_columns]) == 1)
+        if df_nulls.head(1):  # if the filteredDataFrame is not empty
+            raise ValueError(
+                "The key_columns and order_by_columns cannot be null. Please check the quality of the provided columns (null handling)"
+            )
+
+        # check if the order_by columns have the preferred data types
+        recommended_order_by_data_types = [
+            T.TimestampType(),
+            T.TimestampNTZType(),
+            T.DataType(),
+            T.IntegerType(),
+            T.LongType(),
+            T.DoubleType(),
+            T.FloatType(),
+            T.DecimalType(),
+        ]
+
+        for c in context.data.schema:
+            if c.name in order_by_columns and c.dataType not in recommended_order_by_data_types:
+                log_message = (
+                    f"action_name : {self.name} | message : order_by_column `{c.name}` is of type {c.dataType}; "
+                    "recommended data types are {recommended_order_by_data_types}"
+                )
+                self._console_logger.warning(log_message)
+                self._tabular_logger.warning(log_message)
+
+        # sort the order_by columns in the preferred order
+        if descending:
+            order_by_list = [F.col(col_name).desc() for col_name in order_by_columns]
+        else:
+            order_by_list = [F.col(col_name).asc() for col_name in order_by_columns]
+
+        # create the window specification
+        window_specification = Window.partitionBy(key_columns).orderBy(order_by_list)
+
+        # generate a column name that is not in the input dataframe
+        def generate_random_string(length):
+            return "".join(random.choice(string.ascii_uppercase) for _ in range(length))
+
+        row_number_col_name = generate_random_string(20)
+        while row_number_col_name in context.data.columns:
+            row_number_col_name = generate_random_string(20)
+
+        # drop the duplicates
+        df = (
+            context.data.withColumn(row_number_col_name, F.row_number().over(window_specification))
+            .filter(F.col(row_number_col_name) == 1)
+            .drop(row_number_col_name)
+        )
+        return context.from_existing(data=df)

cloe_nessy/pipeline/actions/transform_group_aggregate.py

@@ -0,0 +1,104 @@
+from typing import Any
+
+import pyspark.sql.functions as F
+
+from ..pipeline_action import PipelineAction
+from ..pipeline_context import PipelineContext
+
+
+class TransformGroupAggregate(PipelineAction):
+    """Performs aggregation operations on grouped data within a DataFrame.
+
+    This class allows you to group data by specified columns and apply various aggregation functions
+    to other columns. The aggregation functions can be specified as a dictionary where keys are column names
+    and values are either a single aggregation function or a list of functions.
+
+    Example:
+    ```yaml
+    Transform Group Aggregate:
+        action: TRANSFORM_GROUP_AGGREGATE
+        options:
+            grouping_columns:
+                - column1
+                - column2
+            aggregations:
+                column3:
+                    - sum
+                    - avg
+                column4: max
+    ```
+
+    Attributes:
+        name (str): The name of the action, default is "TRANSFORM_GROUP_AGGREGATE".
+
+    Methods:
+        run(context, grouping_columns=None, aggregations=None, **_):
+            Executes the aggregation on the grouped data.
+
+    Raises:
+        ValueError: If the context data is None.
+        ValueError: If no aggregations are provided.
+        ValueError: If invalid aggregation operations are provided.
+        ValueError: If columns with unsupported data types are included in the aggregations.
+    """
+
+    name: str = "TRANSFORM_GROUP_AGGREGATE"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        grouping_columns: list[str] | None = None,
+        aggregations: dict[str, str | list] | None = None,
+        **_: Any,
+    ) -> PipelineContext:
+        """Executes the aggregation on the grouped data.
+
+        Args:
+            context: The context in which this action is executed.
+            grouping_columns: A list of columns to group by.
+            aggregations: A dictionary where keys are column names and values are either a single
+                aggregation function or a list of functions.
+
+        Raises:
+            ValueError: If the context data is None.
+            ValueError: If no aggregations are provided.
+            ValueError: If invalid aggregation operations are provided.
+            ValueError: If columns with unsupported data types are included in the aggregations.
+
+        Returns:
+            PipelineContext: The context after the execution of this action.
+        """
+        if context.data is None:
+            raise ValueError("Data from the context is required for the operation.")
+
+        if grouping_columns is None:
+            raise ValueError("Please provide at least one grouping column")
+        if aggregations is None:
+            raise ValueError("Please provide aggregations.")
+
+        valid_operations = ["avg", "max", "min", "mean", "sum", "count"]
+
+        for operation in aggregations.values():
+            if isinstance(operation, list):
+                if not set(operation).issubset(valid_operations):
+                    raise ValueError(f"Please provide valid operations. Valid operations are {valid_operations}")
+            elif isinstance(operation, str):
+                if operation not in valid_operations:
+                    raise ValueError(f"Please provide valid operations. Valid operations are {valid_operations}")
+            else:
+                raise ValueError("OPERATION DATATYPE INVALID")
+
+        aggregation_list = []
+        for column_name, aggregation in aggregations.items():
+            if isinstance(aggregation, list):
+                for subaggregation in aggregation:
+                    aggregation_list.append(
+                        getattr(F, subaggregation)(column_name).alias(f"{subaggregation}_{column_name}")
+                    )
+            else:
+                aggregation_list.append(getattr(F, aggregation)(column_name).alias(f"{aggregation}_{column_name}"))
+
+        df = context.data.groupBy(grouping_columns).agg(*aggregation_list)
+
+        return context.from_existing(data=df)

{cloe_nessy-0.2.11.dist-info → cloe_nessy-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cloe-nessy
-Version: 0.2.11
+Version: 0.3.2
 Summary: Your friendly datalake monster.
 Home-page: https://initions.com/
 Author: initions

{cloe_nessy-0.2.11.dist-info → cloe_nessy-0.3.2.dist-info}/RECORD

@@ -47,7 +47,7 @@ cloe_nessy/pipeline/pipeline_config.py,sha256=BN3ZSbr6bC-X9edoh-n5vRfPHFMbgtAU7m
 cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0YEg0zlQTz58,1874
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
-cloe_nessy/pipeline/actions/__init__.py,sha256=shWYl1TDL2f58wHfBhPpiLldreNkvLGJjhnBaTYusFY,2066
+cloe_nessy/pipeline/actions/__init__.py,sha256=Psksv49DVhWHR2D1OuMxvYClF1Vjh5shiyy9yBdWnb0,2160
 cloe_nessy/pipeline/actions/read_api.py,sha256=wGyPZdeh3Cam_BQBilltWBWCIdD9I_kv4lunEhE39Tg,6625
 cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=aZy4sJLLE8ZQ_SPXGSDoHYaBJTz8s7xQDVn5eYrYHvE,2689
 cloe_nessy/pipeline/actions/read_excel.py,sha256=EgHbK1wO6dkDo0KErYDhK_2sNIkIoa-6As9oo9dNFsE,7708
@@ -56,9 +56,11 @@ cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=aZtkstf9jBYYN2MGnazz63B
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=Nz3Ncr-Zd-wy8g9-aN5XcvpWAHLyWs70RpZ7KqKqIaU,1788
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=V0TzeQFpBYur_T1Nv0nRpOU02nKQ2iypo2CCcV2rBtk,3083
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=DmT-29dIqbz_xTj4GSCfnbgYRCiUrWzKvGrRYy1frNw,4004
+cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=2VN5_wza7sD7fERyG6ElGh_Yo-W-Mxw-QBmtDXs1MGQ,5063
 cloe_nessy/pipeline/actions/transform_distinct.py,sha256=sdCElXCM77AQ0m6Zzg_h7cyavBOxo7W9K1NrsvNLufA,1105
 cloe_nessy/pipeline/actions/transform_filter.py,sha256=vOAxKtNWCABLb6G6Xz98NK7fEfgn6QJia31S7IvoUTg,1428
 cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=cli59HCERFge7f0RB8yXw2oDtHSbMCWQMdeCeqhbdg8,2355
+cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=HcY4sqb2yNBCz90jQtxGA8fZPuQXfJuaDmv8lWuoTqg,4050
 cloe_nessy/pipeline/actions/transform_join.py,sha256=qktyaN2kcCkmoH3RILTc-UGYsGACx1nXH6xLtuvYi7k,3080
 cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=xN_cQgHSMSyPsyYXBdoe2i5pHnyH-kkH5do8qr3vybw,4157
 cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=fFdg3353QCE3zBei6iYQW9huPBcQ906sJLioaOUWj3s,1924
@@ -72,7 +74,7 @@ cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEv
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.2.11.dist-info/METADATA,sha256=ggGZFc78AgFrjarO9XcfZAQcqc-mSJKwpwozdk_frlQ,1838
-cloe_nessy-0.2.11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-cloe_nessy-0.2.11.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.2.11.dist-info/RECORD,,
+cloe_nessy-0.3.2.dist-info/METADATA,sha256=7w0f9JC9rm0tmEBYvkPSuTPTKIoGaHFytY7eYf1GRkU,1837
+cloe_nessy-0.3.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+cloe_nessy-0.3.2.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.2.dist-info/RECORD,,