cloe-nessy 0.3.16.3b0__py3-none-any.whl → 0.3.16.6__py3-none-any.whl

cloe_nessy/integration/reader/catalog_reader.py

@@ -50,39 +50,3 @@ class CatalogReader(BaseReader):
             raise ReadOperationFailedError(
                 f"An error occurred while reading the table '{table_identifier}': {err}"
             ) from err
-
-    def read_stream(
-        self, table_identifier: str = "", *, options: dict[str, str] | None = None, **kwargs: Any
-    ) -> DataFrame:
-        """Reads a streaming table from the Unity Catalog.
-
-        Args:
-            table_identifier: The table identifier in the Unity Catalog in the format 'catalog.schema.table'.
-            options: PySpark options for the read stream operation.
-            **kwargs: Additional keyword arguments to maintain compatibility with the base class method.
-
-        Returns:
-            The Spark Streaming DataFrame containing the read data.
-
-        Raises:
-            ValueError: If the table_identifier is not provided, is not a string, or is not in the correct format.
-            Exception: For any other unexpected errors during streaming read operation.
-        """
-        if options is None:
-            options = {}
-        if not table_identifier:
-            raise ValueError("table_identifier is required")
-        if not isinstance(table_identifier, str):
-            raise ValueError("table_identifier must be a string")
-        if len(table_identifier.split(".")) != 3:
-            raise ValueError("table_identifier must be in the format 'catalog.schema.table'")
-
-        try:
-            df = self._spark.readStream.table(table_identifier, **options)
-            return df
-        except AnalysisException as err:
-            raise ValueError(f"Table not found or not streamable: {table_identifier}") from err
-        except Exception as err:
-            raise ReadOperationFailedError(
-                f"An error occurred while reading the stream from table '{table_identifier}': {err}"
-            ) from err

cloe_nessy/integration/writer/catalog_writer.py

@@ -5,7 +5,7 @@ class CatalogWriter:
     """A writer for Catalog tables."""
 
     @staticmethod
-    def write(
+    def write_table(
         df: DataFrame | None,
         table_identifier: str | None,
         partition_by: str | list[str] | None = None,
@@ -46,65 +46,3 @@ class CatalogWriter:
         if options is None:
             options = {}
         df.write.saveAsTable(table_identifier, mode=mode, partitionBy=partition_by, **options)
-
-    @staticmethod
-    def write_stream(
-        df: DataFrame | None,
-        table_identifier: str | None,
-        checkpoint_location: str | None = None,
-        trigger_dict: dict | None = None,
-        options: dict[str, str] | None = None,
-        mode: str = "append",
-        await_termination: bool = False,
-    ) -> None:
-        """Write a streaming DataFrame to a Unity Catalog table.
-
-        Args:
-            df: The streaming DataFrame to write.
-            table_identifier: The table identifier in the Unity Catalog in the
-                              format 'catalog.schema.table'.
-            checkpoint_location: Location for checkpointing. Required for stream recovery.
-            trigger_dict: A dictionary specifying the trigger configuration for the streaming query.
-                Supported keys include:
-                - "processingTime": Specifies a time interval (e.g., "10 seconds") for micro-batch processing.
-                - "once": Processes all available data once and then stops.
-                - "continuous": Specifies a time interval (e.g., "1 second") for continuous processing.
-                - "availableNow": Processes all available data immediately and then stops.
-                If nothing is provided, the default is {"availableNow": True}.
-            options: PySpark options for the DataFrame streaming write operation.
-            mode: The write mode. For streaming, typically "append".
-            await_termination: If True, the function will wait for the streaming
-                query to finish before returning.
-
-        Raises:
-            ValueError: If the mode is not supported for streaming operations.
-            ValueError: If the table_identifier is not a string or not in the format 'catalog.schema.table'.
-            ValueError: If the DataFrame is None.
-            ValueError: If checkpoint_location is not provided.
-        """
-        if mode not in ("append", "complete", "update"):
-            raise ValueError("mode must be one of append, complete, update for streaming operations")
-        if not table_identifier:
-            raise ValueError("table_identifier is required")
-        elif not isinstance(table_identifier, str):
-            raise ValueError("table_identifier must be a string")
-        elif len(table_identifier.split(".")) != 3:
-            raise ValueError("table_identifier must be in the format 'catalog.schema.table'")
-        if not df:
-            raise ValueError("df is required, but was None.")
-        if not checkpoint_location:
-            raise ValueError("checkpoint_location is required for streaming operations")
-
-        if options is None:
-            options = {}
-        if trigger_dict is None:
-            trigger_dict = {"availableNow": True}
-
-        stream_writer = df.writeStream.format("delta").outputMode(mode)
-        stream_writer.options(**options).option("checkpointLocation", checkpoint_location)
-        stream_writer.trigger(**trigger_dict)
-
-        query = stream_writer.toTable(table_identifier)
-
-        if await_termination:
-            query.awaitTermination()

cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py

@@ -196,11 +196,7 @@ class DeltaMergeWriter(BaseDeltaWriter):
 
         config = DeltaMergeConfig(dataframe_columns=data_frame.columns, **kwargs)
 
-        delta_table = self.table_manager.get_delta_table(
-            table=table,
-            location=storage_path,
-            spark=data_frame.sparkSession,
-        )
+        delta_table = self.table_manager.get_delta_table(location=storage_path, spark=data_frame.sparkSession)
 
         match_conditions = self._build_match_conditions(data_frame, config)
 

cloe_nessy/models/column.py

@@ -5,7 +5,6 @@ from pydantic import BaseModel, Field, field_validator, model_validator
 
 COLUMN_DATA_TYPE_LIST = {
     "string",
-    "decimal",
     "integer",
     "int",
     "smallint",
@@ -32,7 +31,7 @@ class Column(BaseModel):
     nullable: bool
     default_value: Any = None
     generated: str | None = None
-    business_properties: dict[str, Any] = Field(default_factory=dict)
+    properties: dict[str, Any] = Field(default_factory=dict)
     comment: str | None = None
 
     @field_validator("data_type", mode="before")
@@ -44,7 +43,7 @@ class Column(BaseModel):
         """
         val = raw.lower()
         base_data_types = re.findall(r"\b[a-z]+\b", val)
-        forbidden_characters = re.findall(r"[^a-z0-9\(\)\<\>,\s]+", val)
+        forbidden_characters = re.findall(r"[^a-z\<\>)]+", val)
 
         if forbidden_characters:
             raise ValueError(f"Forbidden characters in data type definition [ '{val}' ]: [' {forbidden_characters} ']")

cloe_nessy/models/schema.py

@@ -43,7 +43,6 @@ class Schema(ReadInstancesMixin):
             raise FileNotFoundError("Schema file not found.")
 
         schema, schema_errors = super().read_instance_from_file(processed_instance_path)
-        table_errors: list[ValidationErrorType] = []
         if schema:
             schema.storage_path = "" if not schema.storage_path else schema.storage_path
             tables, table_errors = Table.read_instances_from_directory(

cloe_nessy/models/templates/create_table.sql.j2

@@ -13,14 +13,6 @@ USING delta
 {% if table.storage_path %}
 LOCATION '{{ table.storage_path }}'
 {% endif %}
-{% if table.properties %}
-TBLPROPERTIES (
-    {%- for key, value in table.properties.items() %}
-        {%- if not loop.first %}, {% endif -%}
-        '{{key}}' = '{{value}}'
-    {%- endfor -%}
-)
-{% endif %}
 {% if table.partition_by -%}
     {%- if table.liquid_clustering -%} CLUSTER {%- else -%} PARTITIONED {%- endif %} BY (
         {%- for column in table.partition_by -%}
@@ -42,17 +34,3 @@ ALTER TABLE {{ table.escaped_identifier }} ADD CONSTRAINT {{constraint.name}} CH
 {%- if table.comment %}
 COMMENT ON TABLE {{ table.escaped_identifier }} IS '{{ table.comment }}';
 {%- endif %}
-
-{%- if table.business_properties %}
-{%- for tag_key, tag_value in table.business_properties.items() %}
-SET TAG ON TABLE {{ table.escaped_identifier }} "{{ tag_key }}"{% if tag_value %}="{{ tag_value }}"{% endif %};
-{%- endfor %}
-{%- endif %}
-
-{%- for column in table.columns %}
-{%- if column.business_properties %}
-{%- for tag_key, tag_value in column.business_properties.items() %}
-SET TAG ON COLUMN {{ table.escaped_identifier }}.`{{ column.name }}` "{{ tag_key }}"{% if tag_value %}="{{ tag_value }}"{% endif %};
-{%- endfor %}
-{%- endif %}
-{%- endfor %}

cloe_nessy/object_manager/table_manager.py

@@ -186,9 +186,6 @@ class TableManager(LoggerMixin):
     def get_delta_table(self, table: Table | None = None, location: str | None = None, spark=None) -> DeltaTable:
         """Get the DeltaTable object from the Table objects location or a location string.
 
-        For managed tables, uses the table identifier to access the DeltaTable.
-        For external tables or when a location is provided, uses the storage path.
-
         Args:
             table: A Table object representing the Delta table.
             location: A string representing the table location.
@@ -198,34 +195,17 @@ class TableManager(LoggerMixin):
             The DeltaTable object corresponding to the given Table object or location string.
 
         Raises:
-            ValueError: If neither table nor location is provided.
+            ValueError: If neither table nor location is provided, or if both are provided.
         """
-        if table is None and location is None:
-            self._console_logger.error("Invalid parameters: both table and location are None")
-            raise ValueError("Either table or location must be provided.")
-
-        spark_session = spark or self._spark
-
-        if table is not None and location is not None:
-            self._console_logger.info(
-                f"Both table ({table.identifier}) and location ({location}) provided. Using table object as priority."
+        if (table is None and location is None) or (table is not None and location is not None):
+            raise ValueError(
+                f"Either table or location must be provided, but not both. Table: {table}, location: {location}",
             )
 
         if table is not None:
-            if table.is_external is False:
-                self._console_logger.info(f"Getting DeltaTable object for managed table: {table.identifier}")
-                return DeltaTable.forName(spark_session, table.identifier)
-
-            table_location = str(table.storage_path)
-            self._console_logger.info(f"Getting DeltaTable object for external table location: {table_location}")
-            return DeltaTable.forPath(spark_session, table_location)
-
-        self._console_logger.info(f"No table object provided, using location: {location}")
-        if location is None:
-            self._console_logger.error("Location is None - this should not happen!")
-            raise ValueError("Location cannot be None when no table object is provided")
+            location = str(table.storage_path)
         self._console_logger.info(f"Getting DeltaTable object for location: {location}")
-        return DeltaTable.forPath(spark_session, str(location))
+        return DeltaTable.forPath(spark or self._spark, str(location))
 
     def table_exists(self, table: Table | None = None, table_identifier: str | None = None) -> bool:
         """Checks if a table exists in the catalog.
@@ -255,10 +235,9 @@ class TableManager(LoggerMixin):
                 raise ValueError("Invalid table identifier format. Expected 'catalog.schema.table'.")
 
         query_result = self._spark.sql(
-            # Using both upper and lower case to ensure compatibility with case changes in Databricks
             f"""
                 SELECT 1 FROM {catalog}.information_schema.tables
-                WHERE table_name in ('{table_name}', '{table_name.lower()}')
+                WHERE table_name = '{table_name}'
                 AND table_schema = '{schema}'
                 LIMIT 1""",
         )

cloe_nessy/pipeline/actions/read_catalog_table.py

@@ -15,23 +15,13 @@ class ReadCatalogTableAction(PipelineAction):
     into a DataFrame and returned as part of an updated `PipelineContext`.
 
     Example:
-        === "Batch Read"
-            ```yaml
-            Read Sales Table:
-                action: READ_CATALOG_TABLE
-                options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    options: <options for the CatalogReader read method>
-            ```
-        === "Streaming Read"
-            ```yaml
-            Read Sales Table Stream:
-                action: READ_CATALOG_TABLE
-                options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    stream: true
-                    options: <options for the CatalogReader read_stream method>
-            ```
+        ```yaml
+        Read Sales Table:
+            action: READ_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                options: <options for the CatalogReader read method>
+        ```
     """
 
     name: str = "READ_CATALOG_TABLE"
@@ -42,7 +32,6 @@ class ReadCatalogTableAction(PipelineAction):
         *,
         table_identifier: str | None = None,
         options: dict[str, str] | None = None,
-        stream: bool = False,
         **_: Any,  # define kwargs to match the base class signature
     ) -> PipelineContext:
         """Reads a table from Unity Catalog using a specified table identifier and optional reader configurations.
@@ -55,13 +44,7 @@ class ReadCatalogTableAction(PipelineAction):
                 identifier from the `table_metadata` in the `context`.
             options: A dictionary of options for customizing
                 the [`CatalogReader`][cloe_nessy.integration.reader.catalog_reader]
-                behavior, such as filters or reading modes.
-            stream: If True, the action will read the table as a stream.
-            checkpoint_location: The location for storing
-                checkpoints if streaming is enabled.
-            trigger_dict: A dictionary specifying the trigger
-                configuration for the streaming query, such as processing time or
-                continuous processing.
+                behavior, such as filters or reading modes. Defaults to None.
 
         Raises:
             ValueError: If neither `table_identifier` nor `table_metadata.identifier` in the `context` is provided.
@@ -78,10 +61,5 @@ class ReadCatalogTableAction(PipelineAction):
             raise ValueError("Table name must be specified or a valid Table object with identifier must be set.")
 
         table_reader = CatalogReader()
-        if stream:
-            context.runtime_info = (context.runtime_info or {}) | {"streaming": True}
-            df = table_reader.read_stream(table_identifier=table_identifier, options=options)
-        else:
-            df = table_reader.read(table_identifier=table_identifier, options=options)
-
-        return context.from_existing(data=df, runtime_info=context.runtime_info)
+        df = table_reader.read(table_identifier=table_identifier, options=options)
+        return context.from_existing(data=df)

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -1,94 +1,66 @@
-from pathlib import Path
+import pathlib
 from typing import Any
 
-from ...models import Table
+from ...models import Schema
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
 
 class ReadMetadataYAMLAction(PipelineAction):
-    """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+    """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
     Example:
-        === "Managed Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-            ```
-        === "External Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-                    storage_path: abfs://external_storage/sales_data/sales_table
-            ```
+        ```yaml
+        Read Schema Metadata:
+            action: READ_METADATA_YAML_ACTION
+            options:
+                path: excel_file_folder/excel_files_june/
+                file_name: sales_schema.yml
+                table_name: sales
+        ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
 
+    @staticmethod
     def run(
-        self,
         context: PipelineContext,
         *,
-        file_path: str | None = None,
-        catalog_name: str | None = None,
-        schema_name: str | None = None,
-        storage_path: str | None = None,
+        path: str | None = None,
+        file_name: str | None = None,
+        table_name: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
         Args:
             context: The context in which this Action is executed.
-            file_path: The path to the file that defines the table.
-            catalog_name: The name of the catalog for the table.
-            schema_name: The name of the schema for the table.
-            storage_path: The storage path for the table, if applicable. If not
-                provided, the table will be considered a managed table.
+            path: The path to the data contract directory.
+            file_name: The name of the file that defines the schema.
+            table_name: The name of the table for which to retrieve metadata.
 
         Raises:
-            ValueError: If any issues occur while reading the table metadata, such as an invalid table,
-                missing file, missing path, or missing catalog/schema names.
+            ValueError: If any issues occur while reading the schema, such as an invalid schema,
+                missing file, or missing path.
 
         Returns:
             The context after the execution of this Action, containing the table metadata.
         """
-        missing_params = []
-        if not file_path:
-            missing_params.append("file_path")
-        if not catalog_name:
-            missing_params.append("catalog_name")
-        if not schema_name:
-            missing_params.append("schema_name")
+        if not path:
+            raise ValueError("No path provided. Please specify path to schema metadata.")
+        if not file_name:
+            raise ValueError("No file_name provided. Please specify file name.")
+        if not table_name:
+            raise ValueError("No table_name provided. Please specify table name.")
 
-        if missing_params:
-            raise ValueError(
-                f"Missing required parameters: {', '.join(missing_params)}. Please specify all required parameters."
-            )
+        path_obj = pathlib.Path(path)
 
-        final_file_path = Path(file_path) if file_path else Path()
-
-        table, errors = Table.read_instance_from_file(
-            final_file_path,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-        )
+        schema, errors = Schema.read_instance_from_file(path_obj / file_name)
         if errors:
-            raise ValueError(f"Errors while reading table metadata: {errors}")
-        if not table:
-            raise ValueError("No table found in metadata.")
+            raise ValueError(f"Errors while reading schema metadata: {errors}")
+        if not schema:
+            raise ValueError("No schema found in metadata.")
 
-        if not table.storage_path and storage_path:
-            self._console_logger.info(f"Setting storage path for table [ '{table.name}' ] to [ '{storage_path}' ]")
-            table.storage_path = storage_path
-            table.is_external = True
+        table = schema.get_table_by_name(table_name=table_name)
 
-        self._console_logger.info(f"Table [ '{table.name}' ] metadata read successfully from [ '{file_path}' ]")
         return context.from_existing(table_metadata=table)

cloe_nessy/pipeline/actions/transform_join.py

@@ -1,7 +1,5 @@
 from typing import Any
 
-from pyspark.sql import functions as F
-
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 from ..pipeline_step import PipelineStep
@@ -15,74 +13,20 @@ class TransformJoinAction(PipelineAction):
     from [PySpark
     documentation](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 
-    Examples:
-        === "Simple Column Join"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on: id
-                    how: inner
-            ```
-
-        === "Multiple Columns Join"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on: [customer_id, order_date]
-                    how: left
-            ```
-
-        === "Dictionary Join (Different Column Names)"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on:
-                        customer_id: cust_id
-                        order_date: date
-                    how: inner
-            ```
-
-        === "Complex Join with Literals and Expressions"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Load Conditions Table))
-                    join_condition: |
-                        left.material = right.material
-                        AND right.sales_org = '10'
-                        AND right.distr_chan = '10'
-                        AND right.knart = 'ZUVP'
-                        AND right.lovmkond <> 'X'
-                        AND right.sales_unit = 'ST'
-                        AND left.calday BETWEEN
-                            to_date(right.date_from, 'yyyyMMdd') AND
-                            to_date(right.date_to, 'yyyyMMdd')
-                    how: left
-            ```
+    Example:
+        ```yaml
+        Join Tables:
+            action: TRANSFORM_JOIN
+            options:
+                joined_data: ((step:Transform First Table))
+                join_on: id
+                how: anti
+        ```
 
         !!! note "Referencing a DataFrame from another step"
             The `joined_data` parameter is a reference to the DataFrame from another step.
             The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
             for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
-
-        !!! tip "Dictionary Join Syntax"
-            When using a dictionary for `join_on`, the keys represent columns
-            from the DataFrame in context and the values represent columns from
-            the DataFrame in `joined_data`. This is useful when joining tables
-            with different column names for the same logical entity.
-
-        !!! tip "Complex Join Conditions"
-            Use `join_condition` instead of `join_on` for complex joins with literals,
-            expressions, and multiple conditions. Reference columns using `left.column_name`
-            for the main DataFrame and `right.column_name` for the joined DataFrame.
-            Supports all PySpark functions and operators.
     """
 
     name: str = "TRANSFORM_JOIN"
@@ -93,7 +37,6 @@ class TransformJoinAction(PipelineAction):
         *,
         joined_data: PipelineStep | None = None,
         join_on: list[str] | str | dict[str, str] | None = None,
-        join_condition: str | None = None,
         how: str = "inner",
         **_: Any,
     ) -> PipelineContext:
@@ -106,17 +49,13 @@ class TransformJoinAction(PipelineAction):
             join_on: A string for the join column
                 name, a list of column names, or a dictionary mapping columns from the
                 left DataFrame to the right DataFrame. This defines the condition for the
-                join operation. Mutually exclusive with join_condition.
-            join_condition: A string containing a complex join expression with literals,
-                functions, and multiple conditions. Use 'left.' and 'right.' prefixes
-                to reference columns from respective DataFrames. Mutually exclusive with join_on.
+                join operation.
             how: The type of join to perform. Must be one of: inner, cross, outer,
                 full, fullouter, left, leftouter, right, rightouter, semi, anti, etc.
 
         Raises:
             ValueError: If no joined_data is provided.
-            ValueError: If neither join_on nor join_condition is provided.
-            ValueError: If both join_on and join_condition are provided.
+            ValueError: If no join_on is provided.
             ValueError: If the data from context is None.
             ValueError: If the data from the joined_data is None.
 
@@ -125,12 +64,8 @@ class TransformJoinAction(PipelineAction):
         """
         if joined_data is None or joined_data.result is None or joined_data.result.data is None:
             raise ValueError("No joined_data provided.")
-
-        if not join_on and not join_condition:
-            raise ValueError("Either join_on or join_condition must be provided.")
-
-        if join_on and join_condition:
-            raise ValueError("Cannot specify both join_on and join_condition. Use one or the other.")
+        if not join_on:
+            raise ValueError("No join_on provided.")
 
         if context.data is None:
             raise ValueError("Data from the context is required for the operation.")
@@ -138,25 +73,16 @@ class TransformJoinAction(PipelineAction):
         df_right = joined_data.result.data.alias("right")  # type: ignore
         df_left = context.data.alias("left")  # type: ignore
 
-        if join_condition:
-            try:
-                condition = F.expr(join_condition)
-            except Exception as e:
-                # this will not raise an error in most cases, because the evaluation of the expression is lazy
-                raise ValueError(f"Failed to parse join condition '{join_condition}': {str(e)}") from e
-            df = df_left.join(df_right, on=condition, how=how)  # type: ignore
-
-        if join_on:
-            if isinstance(join_on, str):
-                join_condition_list = [join_on]
-            elif isinstance(join_on, list):
-                join_condition_list = join_on
-            else:
-                join_condition_list = [
-                    df_left[left_column] == df_right[right_column]  # type: ignore
-                    for left_column, right_column in join_on.items()
-                ]
-
-            df = df_left.join(df_right, on=join_condition_list, how=how)  # type: ignore
+        if isinstance(join_on, str):
+            join_condition = [join_on]
+        elif isinstance(join_on, list):
+            join_condition = join_on
+        else:
+            join_condition = [
+                df_left[left_column] == df_right[right_column]  # type: ignore
+                for left_column, right_column in join_on.items()
+            ]
+
+        df = df_left.join(df_right, on=join_condition, how=how)  # type: ignore
 
         return context.from_existing(data=df)  # type: ignore

cloe_nessy/pipeline/actions/transform_union.py

@@ -22,8 +22,8 @@ class TransformUnionAction(PipelineAction):
             action: TRANSFORM_UNION
             options:
                 union_data:
-                    - ((step:Filter First Table))
-                    - ((step:SQL Transform Second Table))
+                    - ((step: Filter First Table))
+                    - ((step: SQL Transform Second Table))
         ```
         !!! note "Referencing a DataFrame from another step"
             The `union_data` parameter is a reference to the DataFrame from another step.