cloe-nessy 0.3.3__py3-none-any.whl → 0.3.8__py3-none-any.whl

cloe_nessy/pipeline/actions/transform_concat_columns.py

@@ -10,17 +10,31 @@ class TransformConcatColumnsAction(PipelineAction):
     """Concatenates the specified columns in the given DataFrame.
 
     Example:
-    ```yaml
-    Concat Columns:
-        action: TRANSFORM_CONCAT_COLUMNS
-        options:
-            name: address
-            columns:
-                - street
-                - postcode
-                - country
-            separator: ', '
-    ```
+        === "concat with separator"
+            ```yaml
+            Concat Columns:
+                action: TRANSFORM_CONCAT_COLUMNS
+                options:
+                    name: address
+                    columns:
+                        - street
+                        - postcode
+                        - country
+                    separator: ', '
+            ```
+        === "concat without separator"
+            ```yaml
+            Concat Column:
+                action: TRANSFORM_CONCAT_COLUMNS
+                options:
+                    name: address
+                    columns:
+                        - street
+                        - postcode
+                        - country
+            ```
+            !!! warning "beware of null handling"
+                The `separator` option is not provided, so the default behavior is to use `concat` which returns `NULL` if any of the concatenated values is `NULL`.
     """
 
     name: str = "TRANSFORM_CONCAT_COLUMNS"

cloe_nessy/pipeline/actions/transform_decode.py

@@ -11,13 +11,24 @@ class TransformDecodeAction(PipelineAction):
     """Decodes values of a specified column in the DataFrame based on the given format.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_DECODE
-        options:
-            column: configurations
-            input_format: json
-    ```
+        === "Decode JSON column"
+            ```yaml
+            Expand JSON:
+                action: "TRANSFORM_DECODE"
+                options:
+                    column: "data"
+                    input_format: "json"
+                    schema: "quality INT, timestamp TIMESTAMP, value DOUBLE"
+            ```
+        === "Decode base64 column"
+            ```yaml
+            Decode base64:
+                action: TRANSFORM_DECODE
+                options:
+                    column: encoded_data
+                    input_format: base64
+                    schema: string
+            ```
     """
 
     name: str = "TRANSFORM_DECODE"

cloe_nessy/pipeline/actions/transform_deduplication.py

@@ -18,15 +18,15 @@ class TransformDeduplication(PipelineAction):
     (can be changed to lowest by setting the parameter descending to false).
 
     Example:
-    ```yaml
-    Deduplicate Columns:
-        action: TRANSFORM_DEDUPLICATION
-        options:
-            key_columns:
-                - id
-            order_by_columns:
-                - source_file_modification_time
-    ```
+        ```yaml
+        Deduplicate Columns:
+            action: TRANSFORM_DEDUPLICATION
+            options:
+                key_columns:
+                    - id
+                order_by_columns:
+                    - source_file_modification_time
+        ```
     """
 
     name: str = "TRANSFORM_DEDUPLICATION"

cloe_nessy/pipeline/actions/transform_distinct.py

@@ -10,14 +10,14 @@ class TransformDistinctAction(PipelineAction):
     If a subset is given these columns are used for duplicate comparison. If no subset is given all columns are used.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_DISTINCT
-        options:
-            subset:
-                - first_name
-                - last_name
-    ```
+        ```yaml
+        Distinct Columns:
+            action: TRANSFORM_DISTINCT
+            options:
+                subset:
+                    - first_name
+                    - last_name
+        ```
     """
 
     name: str = "TRANSFORM_DISTINCT"

cloe_nessy/pipeline/actions/transform_filter.py

@@ -8,12 +8,12 @@ class TransformFilterAction(PipelineAction):
     """Filters the DataFrame in the given context based on a specified condition.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_FILTER
-        options:
-            condition: where city="Hamburg"
-    ```
+        ```yaml
+        Filter Columns:
+            action: TRANSFORM_FILTER
+            options:
+                condition: city="Hamburg"
+        ```
     """
 
     name: str = "TRANSFORM_FILTER"

cloe_nessy/pipeline/actions/transform_generic_sql.py

@@ -13,12 +13,18 @@ class TransformSqlAction(PipelineAction):
     statement is executed on that view. The resulting DataFrame is returned.
 
     Example:
-    ```yaml
-    SQL Transform:
-        action: TRANSFORM_SQL
-        options:
-            sql_statement: select city, revenue, firm from {DATA_FRAME} where product="Databricks"
-    ```
+        ```yaml
+        SQL Transform:
+            action: TRANSFORM_SQL
+            options:
+                sql_statement: select city, revenue, firm from {DATA_FRAME} where product="Databricks"
+        ```
+        !!! note
+            The SQL statement should reference the DataFrame as "{DATA_FRAME}".
+            This nessy specific placeholder will be replaced with your input
+            DataFrame from the context. If your pipeline is defined as an
+            f-string, you can escape the curly braces by doubling them, e.g.,
+            "{{DATA_FRAME}}".
     """
 
     name: str = "TRANSFORM_SQL"

cloe_nessy/pipeline/actions/transform_group_aggregate.py

@@ -13,33 +13,27 @@ class TransformGroupAggregate(PipelineAction):
     to other columns. The aggregation functions can be specified as a dictionary where keys are column names
     and values are either a single aggregation function or a list of functions.
 
+    The output DataFrame will contain the grouped columns and the aggregated columns with the aggregation
+    function as a prefix to the column name.
+
     Example:
-    ```yaml
-    Transform Group Aggregate:
-        action: TRANSFORM_GROUP_AGGREGATE
-        options:
-            grouping_columns:
-                - column1
-                - column2
-            aggregations:
-                column3:
-                    - sum
-                    - avg
-                column4: max
-    ```
-
-    Attributes:
-        name (str): The name of the action, default is "TRANSFORM_GROUP_AGGREGATE".
-
-    Methods:
-        run(context, grouping_columns=None, aggregations=None, **_):
-            Executes the aggregation on the grouped data.
-
-    Raises:
-        ValueError: If the context data is None.
-        ValueError: If no aggregations are provided.
-        ValueError: If invalid aggregation operations are provided.
-        ValueError: If columns with unsupported data types are included in the aggregations.
+        ```yaml
+        Transform Group Aggregate:
+            action: TRANSFORM_GROUP_AGGREGATE
+            options:
+                grouping_columns:
+                    - column1
+                    - column2
+                aggregations:
+                    column3:
+                        - sum
+                        - avg
+                    column4: max
+        ```
+
+        This example groups the DataFrame by `column1` and `column2` and aggregates `column3` by sum and average
+        and `column4` by max. The resulting DataFrame will contain the grouped columns `column1` and `column2`
+        and the aggregated columns `sum_column3`, `avg_column3`, and `max_column4`.
     """
 
     name: str = "TRANSFORM_GROUP_AGGREGATE"

cloe_nessy/pipeline/actions/transform_hash_columns.py

@@ -0,0 +1,209 @@
+from typing import Any
+
+from pydantic import BaseModel, Field, model_validator
+from pyspark.sql import functions as F
+
+from ..pipeline_action import PipelineAction
+from ..pipeline_context import PipelineContext
+
+SUPPORTED_ALGORITHMS = {"hash", "md5", "sha1", "sha2", "xxhash64", "crc32"}
+VALID_SHA2_BITS = {224, 256, 384, 512}
+
+
+class HashSettings(BaseModel):
+    """Represents the settings for hashing columns.
+
+    Attributes:
+        columns: List of column names to hash.
+        algorithm: Hashing algorithm to use. Must be one of
+            "hash", "md5", "sha1", "sha2", "xxhash64", or "crc32".
+        bits: Bit length for the 'sha2' algorithm. Optional.
+    """
+
+    columns: list[str]
+    algorithm: str = Field(..., description="Hashing algorithm to use")
+    bits: int | None = Field(default=None, description="Only required for sha2")
+
+    @model_validator(mode="before")
+    def validate_all(cls, values):
+        """Validates the input values for a hashing operation before model instantiation.
+
+        This method performs the following checks:
+
+        1. Ensures the specified hashing algorithm is supported.
+        2. Validates that at least one column is provided and that the columns parameter is a non-empty list.
+        3. Checks that hashing multiple columns is only supported for the 'hash' and 'xxhash64' algorithms.
+        4. For the 'sha2' algorithm, ensures that the 'bits' parameter is one of the valid options.
+        5. Ensures that the 'bits' parameter is not provided for algorithms other than 'sha2'.
+
+        Raises:
+            ValueError: If the algorithm is unsupported, no columns are provided, the columns parameter is invalid,
+                        or the 'bits' parameter is invalid for the specified algorithm.
+            NotImplementedError: If multiple columns are provided and the algorithm does not support hashing multiple columns.
+
+        Args:
+            cls: The class being validated.
+            values: A dictionary of input values containing 'algorithm', 'columns', and 'bits'.
+
+        Returns:
+            The validated input values.
+        """
+        algorithm = values.get("algorithm")
+        columns = values.get("columns")
+        bits = values.get("bits")
+
+        if algorithm not in SUPPORTED_ALGORITHMS:
+            raise ValueError(
+                f"Unsupported hashing algorithm '{algorithm}'. Supported algorithms are: {SUPPORTED_ALGORITHMS}."
+            )
+
+        if not columns or not isinstance(columns, list) or len(columns) == 0:
+            raise ValueError("At least one column must be provided.")
+
+        if len(columns) > 1 and algorithm not in {"hash", "xxhash64"}:
+            raise NotImplementedError(
+                f"Hashing multiple columns is only supported for 'hash' and 'xxhash64'. Algorithm '{algorithm}' does not support this."
+            )
+
+        if algorithm == "sha2":
+            if bits not in VALID_SHA2_BITS:
+                raise ValueError(f"'bits' must be one of {VALID_SHA2_BITS} when using 'sha2'.")
+        elif bits is not None:
+            raise ValueError("'bits' is only allowed when algorithm is 'sha2'.")
+
+        return values
+
+
+class HashConfig(BaseModel):
+    """A configuration model for defining hash settings for specific columns.
+
+    Attributes:
+        hash_config: A dictionary where the keys are column names
+            (as strings) and the values are `HashSettings` objects that define
+            the hash settings for each column.
+
+    Methods:
+        validate_config: Validates the hash configuration to ensure it contains
+            at least one entry and that all column names are valid strings. Raises a
+            `ValueError` if the configuration is invalid.
+    """
+
+    hash_config: dict[str, HashSettings]
+
+    @model_validator(mode="before")
+    def validate_config(cls, values):
+        """Validates the hash configuration provided in the model.
+
+        This method is executed in "before" mode to ensure that the `hash_config`
+        field in the input values meets the required criteria:
+
+        - It must be a dictionary.
+        - It must contain at least one entry.
+        - Each key in the dictionary must be a non-empty string.
+
+        Raises:
+            ValueError: If `hash_config` is missing, not a dictionary, empty, or
+                        contains invalid column names.
+
+        Args:
+            cls: The class to which this validator is applied.
+            values: The input values to validate.
+
+        Returns:
+            The validated input values.
+        """
+        config = values.get("hash_config")
+        if not config or not isinstance(config, dict) or len(config) == 0:
+            raise ValueError("Hash configuration must contain at least one entry.")
+        for new_col in config:
+            if not new_col or not isinstance(new_col, str):
+                raise ValueError(f"Invalid column name '{new_col}' in hash configuration.")
+        return values
+
+
+class TransformHashColumnsAction(PipelineAction):
+    """Hashes specified columns in a DataFrame using a chosen algorithm.
+
+    Given the following `hash_config`:
+
+    Example:
+        ```yaml
+        Hash Columns:
+            action: TRANSFORM_HASH_COLUMNS
+            options:
+                hash_config:
+                - hashed_column1:
+                    columns: ["column1", "column2"]
+                    algorithm: "sha2"
+                    bits: 224
+                - hashed_column2:
+                    columns: ["column1"]
+                    algorithm: "crc32"
+        ```
+
+    Given a DataFrame `df` with the following structure:
+
+    | column1 | column2 | column3 |
+    |---------|---------|---------|
+    |   foo   |   bar   |   baz   |
+
+    After running the action, the resulting DataFrame will look like:
+
+    | column1 | column2 | column3 |                 hashed_column1                            | hashed_column2 |
+    |---------|---------|---------|-----------------------------------------------------------|----------------|
+    |   foo   |   bar   |   baz   |  17725b837e9c896e7123b142eb980131dcc0baa6160db45d4adfdb21 |  1670361220    |
+
+
+    !!! note "Hash values might vary"
+        The actual hash values will depend on the hashing algorithm used and the input data.
+    """
+
+    name: str = "TRANSFORM_HASH_COLUMNS"
+
+    def run(
+        self,
+        context: PipelineContext,
+        *,
+        hash_config: HashConfig | None = None,
+        **_: Any,
+    ) -> PipelineContext:
+        """Hashes the specified columns in the DataFrame.
+
+        Args:
+            context: Context in which this Action is executed.
+            hash_config: Dictionary that contains the configuration for executing the hashing.
+
+        Returns:
+            Updated PipelineContext with hashed columns.
+
+        Raises:
+            ValueError: If columns are missing, data is None, or algorithm/bits are invalid.
+            ValueError: If the hash configuration is invalid.
+        """
+        if context.data is None:
+            raise ValueError("Context data is required for hashing.")
+
+        if not hash_config:
+            raise ValueError("Hash configuration is required.")
+
+        df = context.data
+
+        hash_functions = {
+            "hash": lambda cols: F.hash(*[F.col(c) for c in cols]).cast("string"),
+            "xxhash64": lambda cols: F.xxhash64(F.concat_ws("||", *[F.col(c) for c in cols])).cast("string"),
+            "md5": lambda cols: F.md5(F.concat_ws("||", *[F.col(c) for c in cols])).cast("string"),
+            "sha1": lambda cols: F.sha1(F.concat_ws("||", *[F.col(c) for c in cols])).cast("string"),
+            "sha2": lambda cols, bits: F.sha2(F.concat_ws("||", *[F.col(c) for c in cols]), bits).cast("string"),
+            "crc32": lambda cols: F.crc32(F.concat_ws("||", *[F.col(c) for c in cols])).cast("string"),
+        }
+        default_sha2_bits = 256
+
+        config_obj = HashConfig.model_validate({"hash_config": hash_config})
+        for new_col, config in config_obj.hash_config.items():
+            hash_func = hash_functions[config.algorithm]
+            if config.algorithm == "sha2":
+                df = df.withColumn(new_col, hash_func(config.columns, config.bits or default_sha2_bits))  # type: ignore
+            else:
+                df = df.withColumn(new_col, hash_func(config.columns))  # type: ignore
+
+        return context.from_existing(data=df)

cloe_nessy/pipeline/actions/transform_join.py

@@ -8,18 +8,25 @@ from ..pipeline_step import PipelineStep
 class TransformJoinAction(PipelineAction):
     """Joins the current DataFrame with another DataFrame defined in joined_data.
 
-    The join operation is performed based on specified columns and the type of join
-    indicated by the `how` parameter.
+    The join operation is performed based on specified columns and the type of
+    join indicated by the `how` parameter. Supported join types can be taken
+    from [PySpark
+    documentation](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 
     Example:
-    ```yaml
-    Join Tables:
-        action: TRANSFORM_JOIN
-        options:
-            joined_data: ((step:Transform First Table))
-            join_on: id
-            how: anti
-    ```
+        ```yaml
+        Join Tables:
+            action: TRANSFORM_JOIN
+            options:
+                joined_data: ((step:Transform First Table))
+                join_on: id
+                how: anti
+        ```
+
+        !!! note "Referencing a DataFrame from another step"
+            The `joined_data` parameter is a reference to the DataFrame from another step.
+            The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
+            for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
     """
 
     name: str = "TRANSFORM_JOIN"

cloe_nessy/pipeline/actions/transform_json_normalize.py

@@ -14,12 +14,25 @@ class TransformJsonNormalize(PipelineAction):
     structs are appended after existing columns.
 
     Example:
-    ```yaml
-    Normalize Tables:
-        action: TRANSFORM_JSON_NORMALIZE
-        options:
-            exclude_columns: coordinates
-    ```
+        ```yaml
+        Normalize Tables:
+            action: TRANSFORM_JSON_NORMALIZE
+            options:
+                exclude_columns: coordinates
+        ```
+        Example Input Data:
+
+        | id | name   | coordinates          | attributes                |
+        |----|--------|----------------------|---------------------------|
+        | 1  | Alice  | [10.0, 20.0]         | {"age": 30, "city": "NY"} |
+        | 2  | Bob    | [30.0, 40.0]         | {"age": 25, "city": "LA"} |
+
+        Example Output Data:
+
+        | id | name   | coordinates | attributes_age | attributes_city |
+        |----|--------|-------------|----------------|-----------------|
+        | 1  | Alice  | [10.0, 20.0]| 30             | NY              |
+        | 2  | Bob    | [30.0, 40.0]| 25             | LA              |
     """
 
     name: str = "TRANSFORM_JSON_NORMALIZE"

cloe_nessy/pipeline/actions/transform_rename_columns.py

@@ -12,13 +12,13 @@ class TransformRenameColumnsAction(PipelineAction):
     name and its corresponding value represents the new column name.
 
     Example:
-    ```yaml
-    Rename Column:
-        action: TRANSFORM_RENAME_COLUMNS
-        options:
-            columns:
-                a_very_long_column_name: shortname
-    ```
+        ```yaml
+        Rename Column:
+            action: TRANSFORM_RENAME_COLUMNS
+            options:
+                columns:
+                    a_very_long_column_name: shortname
+        ```
     """
 
     name: str = "TRANSFORM_RENAME_COLUMNS"

cloe_nessy/pipeline/actions/transform_replace_values.py

@@ -13,14 +13,14 @@ class TransformReplaceValuesAction(PipelineAction):
     in the specified columns.
 
     Example:
-    ```yaml
-    Replace Values:
-        action: TRANSFORM_REPLACE_VALUES
-        options:
-            replace:
-                empl_function:
-                    sales_employee: seller
-    ```
+        ```yaml
+        Replace Values:
+            action: TRANSFORM_REPLACE_VALUES
+            options:
+                replace:
+                    empl_function:
+                        sales_employee: seller
+        ```
     """
 
     name: str = "TRANSFORM_REPLACE_VALUES"

cloe_nessy/pipeline/actions/transform_select_columns.py

@@ -14,15 +14,44 @@ class TransformSelectColumnsAction(PipelineAction):
     DataFrame before performing the selection.
 
     Example:
-    ```yaml
-    Select Columns:
-        action: TRANSFORM_SELECT_COLUMNS
-        options:
-            include_columns:
-                - id
-                - city
-                - product
-    ```
+        Example Input Data:
+
+        | id | name   | coordinates          | attributes                |
+        |----|--------|----------------------|---------------------------|
+        | 1  | Alice  | [10.0, 20.0]         | {"age": 30, "city": "NY"} |
+        | 2  | Bob    | [30.0, 40.0]         | {"age": 25, "city": "LA"} |
+        === "Include Columns"
+            ```yaml
+            Select Columns:
+                action: TRANSFORM_SELECT_COLUMNS
+                options:
+                    include_columns:
+                        - id
+                        - name
+                        - coordinates
+            ```
+            Example Output Data:
+
+            | id | name   | coordinates          |
+            |----|--------|----------------------|
+            | 1  | Alice  | [10.0, 20.0]         |
+            | 2  | Bob    | [30.0, 40.0]         |
+
+        === "Exclude Columns"
+            ```yaml
+            Select Columns:
+                action: TRANSFORM_SELECT_COLUMNS
+                options:
+                    exclude_columns:
+                        - coordinates
+            ```
+            Example Output Data:
+
+            | id | name   | attributes                |
+            |----|--------|---------------------------|
+            | 1  | Alice  | {"age": 30, "city": "NY"} |
+            | 2  | Bob    | {"age": 25, "city": "LA"} |
+
     """
 
     name: str = "TRANSFORM_SELECT_COLUMNS"

cloe_nessy/pipeline/actions/transform_union.py

@@ -17,14 +17,18 @@ class TransformUnionAction(PipelineAction):
     empty, a ValueError will be raised.
 
     Example:
-    ```yaml
-    Union Tables:
-        action: TRANSFORM_UNION
-        options:
-            union_data:
-                - ((step: Filter First Table))
-                - ((step: SQL Transform Second Table))
-    ```
+        ```yaml
+        Union Tables:
+            action: TRANSFORM_UNION
+            options:
+                union_data:
+                    - ((step: Filter First Table))
+                    - ((step: SQL Transform Second Table))
+        ```
+        !!! note "Referencing a DataFrame from another step"
+            The `union_data` parameter is a reference to the DataFrame from another step.
+            The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
+            for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
     """
 
     name: str = "TRANSFORM_UNION"

cloe_nessy/pipeline/actions/write_catalog_table.py

@@ -9,15 +9,16 @@ class WriteCatalogTableAction(PipelineAction):
     """Writes a DataFrame to a specified catalog table using [CatalogWriter][cloe_nessy.integration.writer.CatalogWriter].
 
     Example:
-    ```yaml
-    Write Table to Catalog:
-        action: WRITE_CATALOG_TABLE
-        options:
-            table_identifier: my_catalog.business_schema.sales_table
-            mode: append
-            partition_by: day
-            options: <options for the writer>
-    ```
+        ```yaml
+        Write Table to Catalog:
+            action: WRITE_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                mode: append
+                partition_by: day
+                options:
+                    mergeSchema: true
+        ```
     """
 
     name: str = "WRITE_CATALOG_TABLE"
@@ -42,7 +43,7 @@ class WriteCatalogTableAction(PipelineAction):
             mode: The write mode. One of 'append', 'overwrite', 'error',
                 'errorifexists', or 'ignore'.
             partition_by: Names of the partitioning columns.
-            options: Additional options for the write operation.
+            options: PySpark options for the DataFrame.saveAsTable operation (e.g. mergeSchema:true).
 
         Raises:
             ValueError: If the table name is not specified or cannot be inferred from