cloe-nessy 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

cloe_nessy/integration/reader/file_reader.py

@@ -46,7 +46,13 @@ class FileReader(BaseReader):
         if not spark_format and not extension:
             raise ValueError("Either spark_format or extension must be provided.")
         self._console_logger.debug(f"Reading files from [ '{location}' ] ...")
-        extension_to_datatype_dict = {"csv": "csv", "json": "json", "parquet": "parquet", "txt": "text", "xml": "xml"}
+        extension_to_datatype_dict = {
+            "csv": "csv",
+            "json": "json",
+            "parquet": "parquet",
+            "txt": "text",
+            "xml": "xml",
+        }
 
         if extension and not spark_format:
             if extension not in extension_to_datatype_dict:

cloe_nessy/integration/writer/catalog_writer.py

@@ -20,7 +20,7 @@ class CatalogWriter:
                               format 'catalog.schema.table'.
             mode: The write mode. One of append, overwrite, error, errorifexists, ignore.
             partition_by: Names of the partitioning columns.
-            options: All other string options.
+            options: PySpark options for the DataFrame.saveAsTable operation (e.g. mergeSchema:true).
 
         Notes:
             append: Append contents of this DataFrame to existing data.

cloe_nessy/pipeline/actions/read_api.py

@@ -55,51 +55,75 @@ class ReadAPIAction(PipelineAction):
     DataFrame containing the response data.
 
     Example:
-    ```yaml
-    Read API:
-        action: READ_API
-        options:
-            base_url: https://some_url.com/api/
-            endpoint: my/endpoint/
-            method: GET
-            timeout: 90
-            auth:
-                - type: basic
-                  username: my_username
-                  password: my_password
-                - type: secret_scope
-                  secret_scope: my_secret_scope
-                  header_template:
-                    "header_key_1": "<ENVIRONMENT_VARIABLE_NAME>"
-                - type: secret_scope
-                  secret_scope: my_secret_scope
-                  header_template:
-                    "header_key_2": "<SECRET_NAME>"
-                - type: secret_scope
-                  secret_scope: my_other_secret_scope
-                  header_template:
-                    "header_key_3": "<SECRET_NAME>"
-                - type: azure_oauth
-                  client_id: my_client_id
-                  client_secret: my_client_secret
-                  tenant_id: my_tenant_id
-                  scope: <entra-id-client-id>
-    ```
-
-    The above example will combine the headers from the different auth types. The resulting header will look like this:
-
-    ```json
-    {
-        "header_key_1": "value_from_environment_variable",
-        "header_key_2": "value_from_secret",
-        "header_key_3": "value_from_secret",
-        "Authorization": "Bearer <access_token> (from azure_oauth)",
-        "Authorization": "Basic am9obkBleGFtcGxlLmNvbTphYmMxMjM= (from basic)"
-    }
-    ```
-
-    !!! warning
-
+        === "Basic Usage"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+            ```
+        === "Usage with Parameters and Headers"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+                    method: GET
+                    timeout: 90
+                    headers:
+                        key1: value1
+                        key2: value2
+                    params:
+                        key1: value1
+                        key2: value2
+            ```
+        === "Usage with Authentication"
+            ```yaml
+            Read API:
+                action: READ_API
+                options:
+                    base_url: https://some_url.com/api/
+                    endpoint: my/endpoint/
+                    method: GET
+                    timeout: 90
+                    auth:
+                        - type: basic
+                          username: my_username
+                          password: my_password
+                        - type: secret_scope
+                          secret_scope: my_secret_scope
+                          header_template:
+                            "header_key_1": "<ENVIRONMENT_VARIABLE_NAME>"
+                        - type: secret_scope
+                          secret_scope: my_secret_scope
+                          header_template:
+                            "header_key_2": "<SECRET_NAME>"
+                        - type: secret_scope
+                          secret_scope: my_other_secret_scope
+                          header_template:
+                            "header_key_3": "<SECRET_NAME>"
+                        - type: azure_oauth
+                          client_id: my_client_id
+                          client_secret: my_client_secret
+                          tenant_id: my_tenant_id
+                          scope: <entra-id-client-id>
+            ```
+
+            The above example will combine the headers from the different auth types. The resulting header will look like this:
+
+            ```json
+            {
+                "header_key_1": "value_from_environment_variable",
+                "header_key_2": "value_from_secret",
+                "header_key_3": "value_from_secret",
+                "Authorization": "Bearer <access_token> (from azure_oauth)",
+                "Authorization": "Basic am9obkBleGFtcGxlLmNvbTphYmMxMjM= (from basic)"
+            }
+            ```
+
+    !!! warning "Secret information"
         Don't write sensitive information like passwords or tokens directly in the pipeline configuration.
         Use secret scopes or environment variables instead.
     """

cloe_nessy/pipeline/actions/read_catalog_table.py

@@ -15,13 +15,13 @@ class ReadCatalogTableAction(PipelineAction):
     into a DataFrame and returned as part of an updated `PipelineContext`.
 
     Example:
-    ```yaml
-    Read Sales Table:
-        action: READ_CATALOG_TABLE
-        options:
-            table_identifier: my_catalog.business_schema.sales_table
-            options: <options for the reader>
-    ```
+        ```yaml
+        Read Sales Table:
+            action: READ_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                options: <options for the CatalogReader read method>
+        ```
     """
 
     name: str = "READ_CATALOG_TABLE"
@@ -43,8 +43,8 @@ class ReadCatalogTableAction(PipelineAction):
                 read. If not provided, the function will attempt to use the table
                 identifier from the `table_metadata` in the `context`.
             options: A dictionary of options for customizing
-                the catalog reader's behavior, such as filters or reading modes. Defaults
-                to None.
+                the [`CatalogReader`][cloe_nessy.integration.reader.catalog_reader]
+                behavior, such as filters or reading modes. Defaults to None.
 
         Raises:
             ValueError: If neither `table_identifier` nor `table_metadata.identifier` in the `context` is provided.

cloe_nessy/pipeline/actions/read_excel.py

@@ -21,16 +21,20 @@ class ReadExcelAction(PipelineAction):
     the read files can be included in the context.
 
     Example:
-    ```yaml
-    Read Excel Table:
-        action: READ_EXCEL
-        options:
-            file: excel_file_folder/excel_files_june/interesting_excel_file.xlsx
-            usecols:
-                - key_column
-                - interesting_column
-            options: <more options for the reader>
-    ```
+        ```yaml
+        Read Excel Table:
+            action: READ_EXCEL
+            options:
+                file: excel_file_folder/excel_files_june/interesting_excel_file.xlsx
+                usecols:
+                    - key_column
+                    - interesting_column
+                options: <options for the ExcelDataFrameReader read method>
+        ```
+
+    !!! note "More Options"
+        The `READ_EXCEL` action supports additional options that can be passed to the
+        run method. For more information, refer to the method documentation.
     """
 
     name: str = "READ_EXCEL"

cloe_nessy/pipeline/actions/read_files.py

@@ -14,14 +14,47 @@ class ReadFilesAction(PipelineAction):
     location will be read using a DataFrameReader with the specified format.
 
     Example:
-    ```yaml
-    Read Excel Table:
-        action: READ_FILES
-        options:
-            location: excel_file_folder/excel_files_june/
-            search_subdirs: True
-            spark_format: AVRO
-    ```
+        === "Read files specified by spark_format"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: json_file_folder/
+                    search_subdirs: True
+                    spark_format: JSON
+            ```
+            !!! note "Define Spark Format"
+                Use the `spark_format` option to specify the format with which
+                to read the files. Supported formats are e.g., `CSV`, `JSON`,
+                `PARQUET`, `TEXT`, and `XML`.
+
+        === "Read files specified by extension"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: csv_file_folder/
+                    search_subdirs: True
+                    extension: csv
+            ```
+            !!! note "Define Extension"
+                Use the `extension` option to specify the extension of the files
+                to read. If not specified, the `spark_format` will be derived from
+                the extension.
+
+        === "Read files with a specified spark_format AND extension"
+            ```yaml
+            Read Files:
+                action: READ_FILES
+                options:
+                    location: file_folder/
+                    extension: abc_custom_extension  # specifies the files to read
+                    spark_format: CSV  # specifies the format to read the files with
+            ```
+            !!! note "Define both Extension & Spark Format"
+                Use the `extension` option to specify the extension of the files
+                to read. Additionally, use the `spark_format` option to specify
+                the format with which to read the files.
     """
 
     name: str = "READ_FILES"
@@ -47,7 +80,8 @@ class ReadFilesAction(PipelineAction):
             search_subdirs: Recursively search subdirectories for files
                 if an extension is provided.
             extension: The file extension to filter files by.
-            spark_format: The format to use for reading the files.
+            spark_format: The format to use for reading the files. If not provided,
+                it will be deferred from the file extension.
             schema: The schema of the data. If None, schema is obtained from
                 the context metadata.
             add_metadata_column: Whether to include the `__metadata` column with
@@ -65,30 +99,22 @@ class ReadFilesAction(PipelineAction):
             raise ValueError("No location provided. Please specify location to read files from.")
         if not options:
             options = dict()
+        if not spark_format and not extension:
+            raise ValueError("Either spark_format or extension must be provided.")
 
         if (metadata := context.table_metadata) and schema is None:
             schema = metadata.schema
 
         file_reader = FileReader()
-        if extension:
-            df = file_reader.read(
-                location=location,
-                schema=schema,
-                extension=extension,
-                search_subdirs=search_subdirs,
-                options=options,
-                add_metadata_column=add_metadata_column,
-            )
-        elif spark_format:
-            df = file_reader.read(
-                location=location,
-                schema=schema,
-                spark_format=spark_format,
-                options=options,
-                add_metadata_column=add_metadata_column,
-            )
-        else:
-            raise ValueError("Please provide either the 'extension' or 'spark_format'")
+        df = file_reader.read(
+            location=location,
+            schema=schema,
+            extension=extension,
+            spark_format=spark_format,
+            search_subdirs=search_subdirs,
+            options=options,
+            add_metadata_column=add_metadata_column,
+        )
 
         runtime_info = context.runtime_info
 

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -10,14 +10,14 @@ class ReadMetadataYAMLAction(PipelineAction):
     """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
     Example:
-    ```yaml
-    Read Schema Metadata:
-        action: READ_METADATA_YAML_ACTION
-        options:
-            path: excel_file_folder/excel_files_june/
-            file_name: sales_schema.yml
-            table_name: sales
-    ```
+        ```yaml
+        Read Schema Metadata:
+            action: READ_METADATA_YAML_ACTION
+            options:
+                path: excel_file_folder/excel_files_june/
+                file_name: sales_schema.yml
+                table_name: sales
+        ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
@@ -31,7 +31,7 @@ class ReadMetadataYAMLAction(PipelineAction):
         table_name: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads schema metadata from a yaml file using the `Schema` model.
+        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
         Args:
             context: The context in which this Action is executed.

cloe_nessy/pipeline/actions/transform_change_datatype.py

@@ -9,15 +9,20 @@ from ..pipeline_context import PipelineContext
 class TransformChangeDatatypeAction(PipelineAction):
     """Changes the datatypes of specified columns in the given DataFrame.
 
+    !!! note "Data Types"
+        We make use of the PySpark `cast` function to change the data types of
+        the columns. Valid data types can be found in the [PySpark
+        documentation](https://spark.apache.org/docs/3.5.3/sql-ref-datatypes.html).
+
     Example:
-    ```yaml
-    Transform Columns:
-        action: TRANSFORM_CHANGE_DATATYPE
-        options:
-            columns:
-                id: string
-                revenue: long
-    ```
+        ```yaml
+        Cast Columns:
+            action: TRANSFORM_CHANGE_DATATYPE
+            options:
+                columns:
+                    id: string
+                    revenue: long
+        ```
     """
 
     name: str = "TRANSFORM_CHANGE_DATATYPE"

cloe_nessy/pipeline/actions/transform_clean_column_names.py

@@ -15,6 +15,10 @@ class TransformCleanColumnNamesAction(PipelineAction):
     Removes invalid characters from the column names, including the fields of a struct and
     replaces a single leading underscore by a double underscore.
 
+    Invalid characters include:
+        - Any non-word character (anything other than letters, digits, and underscores).
+        - A single leading underscore.
+
     Example:
         ```yaml
         Clean Column Names:

cloe_nessy/pipeline/actions/transform_concat_columns.py

@@ -10,17 +10,31 @@ class TransformConcatColumnsAction(PipelineAction):
     """Concatenates the specified columns in the given DataFrame.
 
     Example:
-    ```yaml
-    Concat Columns:
-        action: TRANSFORM_CONCAT_COLUMNS
-        options:
-            name: address
-            columns:
-                - street
-                - postcode
-                - country
-            separator: ', '
-    ```
+        === "concat with separator"
+            ```yaml
+            Concat Columns:
+                action: TRANSFORM_CONCAT_COLUMNS
+                options:
+                    name: address
+                    columns:
+                        - street
+                        - postcode
+                        - country
+                    separator: ', '
+            ```
+        === "concat without separator"
+            ```yaml
+            Concat Column:
+                action: TRANSFORM_CONCAT_COLUMNS
+                options:
+                    name: address
+                    columns:
+                        - street
+                        - postcode
+                        - country
+            ```
+            !!! warning "beware of null handling"
+                The `separator` option is not provided, so the default behavior is to use `concat` which returns `NULL` if any of the concatenated values is `NULL`.
     """
 
     name: str = "TRANSFORM_CONCAT_COLUMNS"

cloe_nessy/pipeline/actions/transform_decode.py

@@ -11,13 +11,24 @@ class TransformDecodeAction(PipelineAction):
     """Decodes values of a specified column in the DataFrame based on the given format.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_DECODE
-        options:
-            column: configurations
-            input_format: json
-    ```
+        === "Decode JSON column"
+            ```yaml
+            Expand JSON:
+                action: "TRANSFORM_DECODE"
+                options:
+                    column: "data"
+                    input_format: "json"
+                    schema: "quality INT, timestamp TIMESTAMP, value DOUBLE"
+            ```
+        === "Decode base64 column"
+            ```yaml
+            Decode base64:
+                action: TRANSFORM_DECODE
+                options:
+                    column: encoded_data
+                    input_format: base64
+                    schema: string
+            ```
     """
 
     name: str = "TRANSFORM_DECODE"

cloe_nessy/pipeline/actions/transform_deduplication.py

@@ -18,15 +18,15 @@ class TransformDeduplication(PipelineAction):
     (can be changed to lowest by setting the parameter descending to false).
 
     Example:
-    ```yaml
-    Deduplicate Columns:
-        action: TRANSFORM_DEDUPLICATION
-        options:
-            key_columns:
-                - id
-            order_by_columns:
-                - source_file_modification_time
-    ```
+        ```yaml
+        Deduplicate Columns:
+            action: TRANSFORM_DEDUPLICATION
+            options:
+                key_columns:
+                    - id
+                order_by_columns:
+                    - source_file_modification_time
+        ```
     """
 
     name: str = "TRANSFORM_DEDUPLICATION"

cloe_nessy/pipeline/actions/transform_distinct.py

@@ -10,14 +10,14 @@ class TransformDistinctAction(PipelineAction):
     If a subset is given these columns are used for duplicate comparison. If no subset is given all columns are used.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_DISTINCT
-        options:
-            subset:
-                - first_name
-                - last_name
-    ```
+        ```yaml
+        Distinct Columns:
+            action: TRANSFORM_DISTINCT
+            options:
+                subset:
+                    - first_name
+                    - last_name
+        ```
     """
 
     name: str = "TRANSFORM_DISTINCT"

cloe_nessy/pipeline/actions/transform_filter.py

@@ -8,12 +8,12 @@ class TransformFilterAction(PipelineAction):
     """Filters the DataFrame in the given context based on a specified condition.
 
     Example:
-    ```yaml
-    Decode Columns:
-        action: TRANSFORM_FILTER
-        options:
-            condition: where city="Hamburg"
-    ```
+        ```yaml
+        Filter Columns:
+            action: TRANSFORM_FILTER
+            options:
+                condition: city="Hamburg"
+        ```
     """
 
     name: str = "TRANSFORM_FILTER"

cloe_nessy/pipeline/actions/transform_generic_sql.py

@@ -13,12 +13,18 @@ class TransformSqlAction(PipelineAction):
     statement is executed on that view. The resulting DataFrame is returned.
 
     Example:
-    ```yaml
-    SQL Transform:
-        action: TRANSFORM_SQL
-        options:
-            sql_statement: select city, revenue, firm from {DATA_FRAME} where product="Databricks"
-    ```
+        ```yaml
+        SQL Transform:
+            action: TRANSFORM_SQL
+            options:
+                sql_statement: select city, revenue, firm from {DATA_FRAME} where product="Databricks"
+        ```
+        !!! note
+            The SQL statement should reference the DataFrame as "{DATA_FRAME}".
+            This nessy specific placeholder will be replaced with your input
+            DataFrame from the context. If your pipeline is defined as an
+            f-string, you can escape the curly braces by doubling them, e.g.,
+            "{{DATA_FRAME}}".
     """
 
     name: str = "TRANSFORM_SQL"

cloe_nessy/pipeline/actions/transform_group_aggregate.py

@@ -13,33 +13,27 @@ class TransformGroupAggregate(PipelineAction):
     to other columns. The aggregation functions can be specified as a dictionary where keys are column names
     and values are either a single aggregation function or a list of functions.
 
+    The output DataFrame will contain the grouped columns and the aggregated columns with the aggregation
+    function as a prefix to the column name.
+
     Example:
-    ```yaml
-    Transform Group Aggregate:
-        action: TRANSFORM_GROUP_AGGREGATE
-        options:
-            grouping_columns:
-                - column1
-                - column2
-            aggregations:
-                column3:
-                    - sum
-                    - avg
-                column4: max
-    ```
-
-    Attributes:
-        name (str): The name of the action, default is "TRANSFORM_GROUP_AGGREGATE".
-
-    Methods:
-        run(context, grouping_columns=None, aggregations=None, **_):
-            Executes the aggregation on the grouped data.
-
-    Raises:
-        ValueError: If the context data is None.
-        ValueError: If no aggregations are provided.
-        ValueError: If invalid aggregation operations are provided.
-        ValueError: If columns with unsupported data types are included in the aggregations.
+        ```yaml
+        Transform Group Aggregate:
+            action: TRANSFORM_GROUP_AGGREGATE
+            options:
+                grouping_columns:
+                    - column1
+                    - column2
+                aggregations:
+                    column3:
+                        - sum
+                        - avg
+                    column4: max
+        ```
+
+        This example groups the DataFrame by `column1` and `column2` and aggregates `column3` by sum and average
+        and `column4` by max. The resulting DataFrame will contain the grouped columns `column1` and `column2`
+        and the aggregated columns `sum_column3`, `avg_column3`, and `max_column4`.
     """
 
     name: str = "TRANSFORM_GROUP_AGGREGATE"

cloe_nessy/pipeline/actions/transform_join.py

@@ -8,18 +8,25 @@ from ..pipeline_step import PipelineStep
 class TransformJoinAction(PipelineAction):
     """Joins the current DataFrame with another DataFrame defined in joined_data.
 
-    The join operation is performed based on specified columns and the type of join
-    indicated by the `how` parameter.
+    The join operation is performed based on specified columns and the type of
+    join indicated by the `how` parameter. Supported join types can be taken
+    from [PySpark
+    documentation](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 
     Example:
-    ```yaml
-    Join Tables:
-        action: TRANSFORM_JOIN
-        options:
-            joined_data: ((step:Transform First Table))
-            join_on: id
-            how: anti
-    ```
+        ```yaml
+        Join Tables:
+            action: TRANSFORM_JOIN
+            options:
+                joined_data: ((step:Transform First Table))
+                join_on: id
+                how: anti
+        ```
+
+        !!! note "Referencing a DataFrame from another step"
+            The `joined_data` parameter is a reference to the DataFrame from another step.
+            The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
+            for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
     """
 
     name: str = "TRANSFORM_JOIN"

cloe_nessy/pipeline/actions/transform_json_normalize.py

@@ -14,12 +14,25 @@ class TransformJsonNormalize(PipelineAction):
     structs are appended after existing columns.
 
     Example:
-    ```yaml
-    Normalize Tables:
-        action: TRANSFORM_JSON_NORMALIZE
-        options:
-            exclude_columns: coordinates
-    ```
+        ```yaml
+        Normalize Tables:
+            action: TRANSFORM_JSON_NORMALIZE
+            options:
+                exclude_columns: coordinates
+        ```
+        Example Input Data:
+
+        | id | name   | coordinates          | attributes                |
+        |----|--------|----------------------|---------------------------|
+        | 1  | Alice  | [10.0, 20.0]         | {"age": 30, "city": "NY"} |
+        | 2  | Bob    | [30.0, 40.0]         | {"age": 25, "city": "LA"} |
+
+        Example Output Data:
+
+        | id | name   | coordinates | attributes_age | attributes_city |
+        |----|--------|-------------|----------------|-----------------|
+        | 1  | Alice  | [10.0, 20.0]| 30             | NY              |
+        | 2  | Bob    | [30.0, 40.0]| 25             | LA              |
     """
 
     name: str = "TRANSFORM_JSON_NORMALIZE"

cloe_nessy/pipeline/actions/transform_rename_columns.py

@@ -12,13 +12,13 @@ class TransformRenameColumnsAction(PipelineAction):
     name and its corresponding value represents the new column name.
 
     Example:
-    ```yaml
-    Rename Column:
-        action: TRANSFORM_RENAME_COLUMNS
-        options:
-            columns:
-                a_very_long_column_name: shortname
-    ```
+        ```yaml
+        Rename Column:
+            action: TRANSFORM_RENAME_COLUMNS
+            options:
+                columns:
+                    a_very_long_column_name: shortname
+        ```
     """
 
     name: str = "TRANSFORM_RENAME_COLUMNS"

cloe_nessy/pipeline/actions/transform_replace_values.py

@@ -13,14 +13,14 @@ class TransformReplaceValuesAction(PipelineAction):
     in the specified columns.
 
     Example:
-    ```yaml
-    Replace Values:
-        action: TRANSFORM_REPLACE_VALUES
-        options:
-            replace:
-                empl_function:
-                    sales_employee: seller
-    ```
+        ```yaml
+        Replace Values:
+            action: TRANSFORM_REPLACE_VALUES
+            options:
+                replace:
+                    empl_function:
+                        sales_employee: seller
+        ```
     """
 
     name: str = "TRANSFORM_REPLACE_VALUES"

cloe_nessy/pipeline/actions/transform_select_columns.py

@@ -14,15 +14,44 @@ class TransformSelectColumnsAction(PipelineAction):
     DataFrame before performing the selection.
 
     Example:
-    ```yaml
-    Select Columns:
-        action: TRANSFORM_SELECT_COLUMNS
-        options:
-            include_columns:
-                - id
-                - city
-                - product
-    ```
+        Example Input Data:
+
+        | id | name   | coordinates          | attributes                |
+        |----|--------|----------------------|---------------------------|
+        | 1  | Alice  | [10.0, 20.0]         | {"age": 30, "city": "NY"} |
+        | 2  | Bob    | [30.0, 40.0]         | {"age": 25, "city": "LA"} |
+        === "Include Columns"
+            ```yaml
+            Select Columns:
+                action: TRANSFORM_SELECT_COLUMNS
+                options:
+                    include_columns:
+                        - id
+                        - name
+                        - coordinates
+            ```
+            Example Output Data:
+
+            | id | name   | coordinates          |
+            |----|--------|----------------------|
+            | 1  | Alice  | [10.0, 20.0]         |
+            | 2  | Bob    | [30.0, 40.0]         |
+
+        === "Exclude Columns"
+            ```yaml
+            Select Columns:
+                action: TRANSFORM_SELECT_COLUMNS
+                options:
+                    exclude_columns:
+                        - coordinates
+            ```
+            Example Output Data:
+
+            | id | name   | attributes                |
+            |----|--------|---------------------------|
+            | 1  | Alice  | {"age": 30, "city": "NY"} |
+            | 2  | Bob    | {"age": 25, "city": "LA"} |
+
     """
 
     name: str = "TRANSFORM_SELECT_COLUMNS"

cloe_nessy/pipeline/actions/transform_union.py

@@ -17,14 +17,18 @@ class TransformUnionAction(PipelineAction):
     empty, a ValueError will be raised.
 
     Example:
-    ```yaml
-    Union Tables:
-        action: TRANSFORM_UNION
-        options:
-            union_data:
-                - ((step: Filter First Table))
-                - ((step: SQL Transform Second Table))
-    ```
+        ```yaml
+        Union Tables:
+            action: TRANSFORM_UNION
+            options:
+                union_data:
+                    - ((step: Filter First Table))
+                    - ((step: SQL Transform Second Table))
+        ```
+        !!! note "Referencing a DataFrame from another step"
+            The `union_data` parameter is a reference to the DataFrame from another step.
+            The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
+            for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
     """
 
     name: str = "TRANSFORM_UNION"

cloe_nessy/pipeline/actions/write_catalog_table.py

@@ -9,15 +9,16 @@ class WriteCatalogTableAction(PipelineAction):
     """Writes a DataFrame to a specified catalog table using [CatalogWriter][cloe_nessy.integration.writer.CatalogWriter].
 
     Example:
-    ```yaml
-    Write Table to Catalog:
-        action: WRITE_CATALOG_TABLE
-        options:
-            table_identifier: my_catalog.business_schema.sales_table
-            mode: append
-            partition_by: day
-            options: <options for the writer>
-    ```
+        ```yaml
+        Write Table to Catalog:
+            action: WRITE_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                mode: append
+                partition_by: day
+                options:
+                    mergeSchema: true
+        ```
     """
 
     name: str = "WRITE_CATALOG_TABLE"
@@ -42,7 +43,7 @@ class WriteCatalogTableAction(PipelineAction):
             mode: The write mode. One of 'append', 'overwrite', 'error',
                 'errorifexists', or 'ignore'.
             partition_by: Names of the partitioning columns.
-            options: Additional options for the write operation.
+            options: PySpark options for the DataFrame.saveAsTable operation (e.g. mergeSchema:true).
 
         Raises:
             ValueError: If the table name is not specified or cannot be inferred from

{cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cloe-nessy
-Version: 0.3.3
+Version: 0.3.5
 Summary: Your friendly datalake monster.
 Home-page: https://initions.com/
 Author: initions

{cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.5.dist-info}/RECORD

@@ -22,10 +22,10 @@ cloe_nessy/integration/reader/api_reader.py,sha256=j3Z5O1oH-Zc43TyA_aYtnDNYC9xFM
 cloe_nessy/integration/reader/catalog_reader.py,sha256=tGK-Y0jZQGOrF9eZUzSr7ils-L58uex6qH9PZ81ZLy8,1835
 cloe_nessy/integration/reader/excel_reader.py,sha256=4kifpIakHpGmap0-P0SUgjJoQdY-eeiZBIDrQp87wK8,8012
 cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
-cloe_nessy/integration/reader/file_reader.py,sha256=pkrW_N5avqQpqcZuIQgHw5CFf7DFpSuKvq88zPZPfyY,3879
+cloe_nessy/integration/reader/file_reader.py,sha256=1os8pZIXAGTJBZjGREmHOTlZeabbikC7sDv5xn3bIjE,3950
 cloe_nessy/integration/reader/reader.py,sha256=e2KVPePQme8SBQJEbL-3zpGasOgTiEvKFTslow2wGPw,1034
 cloe_nessy/integration/writer/__init__.py,sha256=NIh0t1RYlG3J1Y5_CvnR36N9tISmcElD5Tq06ksmqoA,71
-cloe_nessy/integration/writer/catalog_writer.py,sha256=49lDvYttUY79Ye_OMN2cji7lGJNNML4TTsjY7VvLVfc,2137
+cloe_nessy/integration/writer/catalog_writer.py,sha256=Gb-hMdADgO_uUJ7mZPHBYyNme2qXsdFFnzwo7GcShHM,2192
 cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
 cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
 cloe_nessy/models/__init__.py,sha256=_JPN_R5-QDfjYzvrvZDdeOezl0C-JTG-Rk4S1VE5vJM,242
@@ -48,34 +48,34 @@ cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0Y
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
 cloe_nessy/pipeline/actions/__init__.py,sha256=LwKctXy4Jun52BnCVGvWa8nnKVjTSov4GT58j6Zy8zg,2273
-cloe_nessy/pipeline/actions/read_api.py,sha256=wGyPZdeh3Cam_BQBilltWBWCIdD9I_kv4lunEhE39Tg,6625
-cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=aZy4sJLLE8ZQ_SPXGSDoHYaBJTz8s7xQDVn5eYrYHvE,2689
-cloe_nessy/pipeline/actions/read_excel.py,sha256=EgHbK1wO6dkDo0KErYDhK_2sNIkIoa-6As9oo9dNFsE,7708
-cloe_nessy/pipeline/actions/read_files.py,sha256=8twjprqKYEmVu5QITEGe4no45TfhgzZosTFVQ89vV6g,3861
-cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=aZtkstf9jBYYN2MGnazz63BG_hJ7mIgAfKiNqUpc26E,2235
-cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=Nz3Ncr-Zd-wy8g9-aN5XcvpWAHLyWs70RpZ7KqKqIaU,1788
-cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=XuVAVEbp-UiF8PO6wAEJyl1TYgBD7MSnuOGhuEvXKv4,2881
-cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=V0TzeQFpBYur_T1Nv0nRpOU02nKQ2iypo2CCcV2rBtk,3083
-cloe_nessy/pipeline/actions/transform_decode.py,sha256=DmT-29dIqbz_xTj4GSCfnbgYRCiUrWzKvGrRYy1frNw,4004
-cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=2VN5_wza7sD7fERyG6ElGh_Yo-W-Mxw-QBmtDXs1MGQ,5063
-cloe_nessy/pipeline/actions/transform_distinct.py,sha256=R0Wv_YnWOw198r0rPR_72fgH5sp7upgjZzfOPTZ1oPA,1942
-cloe_nessy/pipeline/actions/transform_filter.py,sha256=vOAxKtNWCABLb6G6Xz98NK7fEfgn6QJia31S7IvoUTg,1428
-cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=cli59HCERFge7f0RB8yXw2oDtHSbMCWQMdeCeqhbdg8,2355
-cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=HcY4sqb2yNBCz90jQtxGA8fZPuQXfJuaDmv8lWuoTqg,4050
-cloe_nessy/pipeline/actions/transform_join.py,sha256=qktyaN2kcCkmoH3RILTc-UGYsGACx1nXH6xLtuvYi7k,3080
-cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=xN_cQgHSMSyPsyYXBdoe2i5pHnyH-kkH5do8qr3vybw,4157
-cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=fFdg3353QCE3zBei6iYQW9huPBcQ906sJLioaOUWj3s,1924
-cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=-uOAbHkQZ2X23GB15W4-miAoHzyFH9hJyc6Y_5PA0w8,2017
-cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=Kez8puDK7cRfhleBEX-B-elKCvNPRU9ERSWs9afMGO8,3369
-cloe_nessy/pipeline/actions/transform_union.py,sha256=TDER06IABzxvIez4bGLKCLaDA4eScpTzYRbfUzwv_RQ,2342
-cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=6yAHTX5kZviumgBW_NYVGAUin6U2nDzmic9of6wA8FY,2590
+cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
+cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=-k2wezkv8bE_xwoW7WM1ORhrCXQagKTUuXkhI2ZEROs,2783
+cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
+cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
+cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
+cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
+cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=-CEdcXb7Fz5DQNitGlJ8EVBE_LzxfsInyCIO-D7b4iY,3042
+cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
+cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
+cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
+cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
+cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1QeatjdEis0up4I7cOWBdyo,1446
+cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=_naWfmPdYAUKjPNeHu5qJAohOL7DHCSYz_kwoeRv3OI,2741
+cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpbsPEJkzea5zFJA6MuyjNpOsFud9o,4045
+cloe_nessy/pipeline/actions/transform_join.py,sha256=e_tvMk8YJTAWcUK_EmOgNt0s31ICZoMX_MKOTWx4lBY,3645
+cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
+cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
+cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=1OPHTrjcphfyGepcO7ozYfeqfwA18pjlyHpVKUS_AAU,2049
+cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm3BG9BX4_qUDJMbpj1DsKPY046w,4574
+cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
+cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
 cloe_nessy/session/session_manager.py,sha256=PK7awMc6fmot7f9FMmvIUbIzKFgjcy2o2bZS9kjVs10,6733
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.3.dist-info/METADATA,sha256=StCfl2I5dItaMzO10u3gQw6WxfjZUZHRIodEvKuQu_s,1837
-cloe_nessy-0.3.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-cloe_nessy-0.3.3.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.3.dist-info/RECORD,,
+cloe_nessy-0.3.5.dist-info/METADATA,sha256=UUx3aIUgvCLn7j3H4DbCL1k9-47HPKaANiMQsUj66wo,1837
+cloe_nessy-0.3.5.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+cloe_nessy-0.3.5.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.5.dist-info/RECORD,,

{cloe_nessy-0.3.3.dist-info → cloe_nessy-0.3.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (76.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any