cloe-nessy 0.3.16.5b0__py3-none-any.whl → 0.3.16.6__py3-none-any.whl

cloe_nessy/integration/reader/catalog_reader.py

@@ -50,39 +50,3 @@ class CatalogReader(BaseReader):
             raise ReadOperationFailedError(
                 f"An error occurred while reading the table '{table_identifier}': {err}"
             ) from err
-
-    def read_stream(
-        self, table_identifier: str = "", *, options: dict[str, str] | None = None, **kwargs: Any
-    ) -> DataFrame:
-        """Reads a streaming table from the Unity Catalog.
-
-        Args:
-            table_identifier: The table identifier in the Unity Catalog in the format 'catalog.schema.table'.
-            options: PySpark options for the read stream operation.
-            **kwargs: Additional keyword arguments to maintain compatibility with the base class method.
-
-        Returns:
-            The Spark Streaming DataFrame containing the read data.
-
-        Raises:
-            ValueError: If the table_identifier is not provided, is not a string, or is not in the correct format.
-            Exception: For any other unexpected errors during streaming read operation.
-        """
-        if options is None:
-            options = {}
-        if not table_identifier:
-            raise ValueError("table_identifier is required")
-        if not isinstance(table_identifier, str):
-            raise ValueError("table_identifier must be a string")
-        if len(table_identifier.split(".")) != 3:
-            raise ValueError("table_identifier must be in the format 'catalog.schema.table'")
-
-        try:
-            df = self._spark.readStream.table(table_identifier, **options)
-            return df
-        except AnalysisException as err:
-            raise ValueError(f"Table not found or not streamable: {table_identifier}") from err
-        except Exception as err:
-            raise ReadOperationFailedError(
-                f"An error occurred while reading the stream from table '{table_identifier}': {err}"
-            ) from err

cloe_nessy/integration/writer/catalog_writer.py

@@ -5,7 +5,7 @@ class CatalogWriter:
     """A writer for Catalog tables."""
 
     @staticmethod
-    def write(
+    def write_table(
         df: DataFrame | None,
         table_identifier: str | None,
         partition_by: str | list[str] | None = None,
@@ -46,65 +46,3 @@ class CatalogWriter:
         if options is None:
             options = {}
         df.write.saveAsTable(table_identifier, mode=mode, partitionBy=partition_by, **options)
-
-    @staticmethod
-    def write_stream(
-        df: DataFrame | None,
-        table_identifier: str | None,
-        checkpoint_location: str | None = None,
-        trigger_dict: dict | None = None,
-        options: dict[str, str] | None = None,
-        mode: str = "append",
-        await_termination: bool = False,
-    ) -> None:
-        """Write a streaming DataFrame to a Unity Catalog table.
-
-        Args:
-            df: The streaming DataFrame to write.
-            table_identifier: The table identifier in the Unity Catalog in the
-                              format 'catalog.schema.table'.
-            checkpoint_location: Location for checkpointing. Required for stream recovery.
-            trigger_dict: A dictionary specifying the trigger configuration for the streaming query.
-                Supported keys include:
-                - "processingTime": Specifies a time interval (e.g., "10 seconds") for micro-batch processing.
-                - "once": Processes all available data once and then stops.
-                - "continuous": Specifies a time interval (e.g., "1 second") for continuous processing.
-                - "availableNow": Processes all available data immediately and then stops.
-                If nothing is provided, the default is {"availableNow": True}.
-            options: PySpark options for the DataFrame streaming write operation.
-            mode: The write mode. For streaming, typically "append".
-            await_termination: If True, the function will wait for the streaming
-                query to finish before returning.
-
-        Raises:
-            ValueError: If the mode is not supported for streaming operations.
-            ValueError: If the table_identifier is not a string or not in the format 'catalog.schema.table'.
-            ValueError: If the DataFrame is None.
-            ValueError: If checkpoint_location is not provided.
-        """
-        if mode not in ("append", "complete", "update"):
-            raise ValueError("mode must be one of append, complete, update for streaming operations")
-        if not table_identifier:
-            raise ValueError("table_identifier is required")
-        elif not isinstance(table_identifier, str):
-            raise ValueError("table_identifier must be a string")
-        elif len(table_identifier.split(".")) != 3:
-            raise ValueError("table_identifier must be in the format 'catalog.schema.table'")
-        if not df:
-            raise ValueError("df is required, but was None.")
-        if not checkpoint_location:
-            raise ValueError("checkpoint_location is required for streaming operations")
-
-        if options is None:
-            options = {}
-        if trigger_dict is None:
-            trigger_dict = {"availableNow": True}
-
-        stream_writer = df.writeStream.format("delta").outputMode(mode)
-        stream_writer.options(**options).option("checkpointLocation", checkpoint_location)
-        stream_writer.trigger(**trigger_dict)
-
-        query = stream_writer.toTable(table_identifier)
-
-        if await_termination:
-            query.awaitTermination()

cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py

@@ -196,11 +196,7 @@ class DeltaMergeWriter(BaseDeltaWriter):
 
         config = DeltaMergeConfig(dataframe_columns=data_frame.columns, **kwargs)
 
-        delta_table = self.table_manager.get_delta_table(
-            table=table,
-            location=storage_path,
-            spark=data_frame.sparkSession,
-        )
+        delta_table = self.table_manager.get_delta_table(location=storage_path, spark=data_frame.sparkSession)
 
         match_conditions = self._build_match_conditions(data_frame, config)
 

cloe_nessy/models/column.py

@@ -5,7 +5,6 @@ from pydantic import BaseModel, Field, field_validator, model_validator
 
 COLUMN_DATA_TYPE_LIST = {
     "string",
-    "decimal",
     "integer",
     "int",
     "smallint",
@@ -32,7 +31,7 @@ class Column(BaseModel):
     nullable: bool
     default_value: Any = None
     generated: str | None = None
-    business_properties: dict[str, Any] = Field(default_factory=dict)
+    properties: dict[str, Any] = Field(default_factory=dict)
     comment: str | None = None
 
     @field_validator("data_type", mode="before")
@@ -44,7 +43,7 @@ class Column(BaseModel):
         """
         val = raw.lower()
         base_data_types = re.findall(r"\b[a-z]+\b", val)
-        forbidden_characters = re.findall(r"[^a-z0-9\(\)\<\>,\s]+", val)
+        forbidden_characters = re.findall(r"[^a-z\<\>)]+", val)
 
         if forbidden_characters:
             raise ValueError(f"Forbidden characters in data type definition [ '{val}' ]: [' {forbidden_characters} ']")

cloe_nessy/models/schema.py

@@ -43,7 +43,6 @@ class Schema(ReadInstancesMixin):
             raise FileNotFoundError("Schema file not found.")
 
         schema, schema_errors = super().read_instance_from_file(processed_instance_path)
-        table_errors: list[ValidationErrorType] = []
         if schema:
             schema.storage_path = "" if not schema.storage_path else schema.storage_path
             tables, table_errors = Table.read_instances_from_directory(

cloe_nessy/models/templates/create_table.sql.j2

@@ -13,14 +13,6 @@ USING delta
 {% if table.storage_path %}
 LOCATION '{{ table.storage_path }}'
 {% endif %}
-{% if table.properties %}
-TBLPROPERTIES (
-    {%- for key, value in table.properties.items() %}
-        {%- if not loop.first %}, {% endif -%}
-        '{{key}}' = '{{value}}'
-    {%- endfor -%}
-)
-{% endif %}
 {% if table.partition_by -%}
     {%- if table.liquid_clustering -%} CLUSTER {%- else -%} PARTITIONED {%- endif %} BY (
         {%- for column in table.partition_by -%}
@@ -42,17 +34,3 @@ ALTER TABLE {{ table.escaped_identifier }} ADD CONSTRAINT {{constraint.name}} CH
 {%- if table.comment %}
 COMMENT ON TABLE {{ table.escaped_identifier }} IS '{{ table.comment }}';
 {%- endif %}
-{# Tags do not yet work in Databricks
-{%- if table.business_properties %}
-{%- for tag_key, tag_value in table.business_properties.items() %}
-SET TAG ON TABLE {{ table.escaped_identifier }} `{{ tag_key }}`{% if tag_value %} = `{{ tag_value }}`{% endif %};
-{%- endfor %}
-{%- endif %}
-
-{%- for column in table.columns %}
-{%- if column.business_properties %}
-{%- for tag_key, tag_value in column.business_properties.items() %}
-SET TAG ON COLUMN {{ table.escaped_identifier }}.`{{ column.name }}` `{{ tag_key }}`{% if tag_value %} = `{{ tag_value }}`{% endif %};
-{%- endfor %}
-{%- endif %}
-{%- endfor %} #}

cloe_nessy/object_manager/table_manager.py

@@ -186,9 +186,6 @@ class TableManager(LoggerMixin):
     def get_delta_table(self, table: Table | None = None, location: str | None = None, spark=None) -> DeltaTable:
         """Get the DeltaTable object from the Table objects location or a location string.
 
-        For managed tables, uses the table identifier to access the DeltaTable.
-        For external tables or when a location is provided, uses the storage path.
-
         Args:
             table: A Table object representing the Delta table.
             location: A string representing the table location.
@@ -198,34 +195,17 @@ class TableManager(LoggerMixin):
             The DeltaTable object corresponding to the given Table object or location string.
 
         Raises:
-            ValueError: If neither table nor location is provided.
+            ValueError: If neither table nor location is provided, or if both are provided.
         """
-        if table is None and location is None:
-            self._console_logger.error("Invalid parameters: both table and location are None")
-            raise ValueError("Either table or location must be provided.")
-
-        spark_session = spark or self._spark
-
-        if table is not None and location is not None:
-            self._console_logger.info(
-                f"Both table ({table.identifier}) and location ({location}) provided. Using table object as priority."
+        if (table is None and location is None) or (table is not None and location is not None):
+            raise ValueError(
+                f"Either table or location must be provided, but not both. Table: {table}, location: {location}",
             )
 
         if table is not None:
-            if table.is_external is False:
-                self._console_logger.info(f"Getting DeltaTable object for managed table: {table.identifier}")
-                return DeltaTable.forName(spark_session, table.identifier)
-
-            table_location = str(table.storage_path)
-            self._console_logger.info(f"Getting DeltaTable object for external table location: {table_location}")
-            return DeltaTable.forPath(spark_session, table_location)
-
-        self._console_logger.info(f"No table object provided, using location: {location}")
-        if location is None:
-            self._console_logger.error("Location is None - this should not happen!")
-            raise ValueError("Location cannot be None when no table object is provided")
+            location = str(table.storage_path)
         self._console_logger.info(f"Getting DeltaTable object for location: {location}")
-        return DeltaTable.forPath(spark_session, str(location))
+        return DeltaTable.forPath(spark or self._spark, str(location))
 
     def table_exists(self, table: Table | None = None, table_identifier: str | None = None) -> bool:
         """Checks if a table exists in the catalog.
@@ -255,10 +235,9 @@ class TableManager(LoggerMixin):
                 raise ValueError("Invalid table identifier format. Expected 'catalog.schema.table'.")
 
         query_result = self._spark.sql(
-            # Using both upper and lower case to ensure compatibility with case changes in Databricks
             f"""
                 SELECT 1 FROM {catalog}.information_schema.tables
-                WHERE table_name in ('{table_name}', '{table_name.lower()}')
+                WHERE table_name = '{table_name}'
                 AND table_schema = '{schema}'
                 LIMIT 1""",
         )

cloe_nessy/pipeline/actions/read_catalog_table.py

@@ -15,23 +15,13 @@ class ReadCatalogTableAction(PipelineAction):
     into a DataFrame and returned as part of an updated `PipelineContext`.
 
     Example:
-        === "Batch Read"
-            ```yaml
-            Read Sales Table:
-                action: READ_CATALOG_TABLE
-                options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    options: <options for the CatalogReader read method>
-            ```
-        === "Streaming Read"
-            ```yaml
-            Read Sales Table Stream:
-                action: READ_CATALOG_TABLE
-                options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    stream: true
-                    options: <options for the CatalogReader read_stream method>
-            ```
+        ```yaml
+        Read Sales Table:
+            action: READ_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                options: <options for the CatalogReader read method>
+        ```
     """
 
     name: str = "READ_CATALOG_TABLE"
@@ -42,7 +32,6 @@ class ReadCatalogTableAction(PipelineAction):
         *,
         table_identifier: str | None = None,
         options: dict[str, str] | None = None,
-        stream: bool = False,
         **_: Any,  # define kwargs to match the base class signature
     ) -> PipelineContext:
         """Reads a table from Unity Catalog using a specified table identifier and optional reader configurations.
@@ -55,13 +44,7 @@ class ReadCatalogTableAction(PipelineAction):
                 identifier from the `table_metadata` in the `context`.
             options: A dictionary of options for customizing
                 the [`CatalogReader`][cloe_nessy.integration.reader.catalog_reader]
-                behavior, such as filters or reading modes.
-            stream: If True, the action will read the table as a stream.
-            checkpoint_location: The location for storing
-                checkpoints if streaming is enabled.
-            trigger_dict: A dictionary specifying the trigger
-                configuration for the streaming query, such as processing time or
-                continuous processing.
+                behavior, such as filters or reading modes. Defaults to None.
 
         Raises:
             ValueError: If neither `table_identifier` nor `table_metadata.identifier` in the `context` is provided.
@@ -78,10 +61,5 @@ class ReadCatalogTableAction(PipelineAction):
             raise ValueError("Table name must be specified or a valid Table object with identifier must be set.")
 
         table_reader = CatalogReader()
-        if stream:
-            context.runtime_info = (context.runtime_info or {}) | {"streaming": True}
-            df = table_reader.read_stream(table_identifier=table_identifier, options=options)
-        else:
-            df = table_reader.read(table_identifier=table_identifier, options=options)
-
-        return context.from_existing(data=df, runtime_info=context.runtime_info)
+        df = table_reader.read(table_identifier=table_identifier, options=options)
+        return context.from_existing(data=df)

cloe_nessy/pipeline/actions/read_metadata_yaml.py

@@ -1,94 +1,66 @@
-from pathlib import Path
+import pathlib
 from typing import Any
 
-from ...models import Table
+from ...models import Schema
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
 
 class ReadMetadataYAMLAction(PipelineAction):
-    """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+    """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
     Example:
-        === "Managed Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-            ```
-        === "External Table"
-            ```yaml
-            Read Table Metadata:
-                action: READ_METADATA_YAML_ACTION
-                options:
-                    file_path: metadata/schemas/bronze/sales_table.yml
-                    catalog_name: production
-                    schema_name: sales_data
-                    storage_path: abfs://external_storage/sales_data/sales_table
-            ```
+        ```yaml
+        Read Schema Metadata:
+            action: READ_METADATA_YAML_ACTION
+            options:
+                path: excel_file_folder/excel_files_june/
+                file_name: sales_schema.yml
+                table_name: sales
+        ```
     """
 
     name: str = "READ_METADATA_YAML_ACTION"
 
+    @staticmethod
     def run(
-        self,
         context: PipelineContext,
         *,
-        file_path: str | None = None,
-        catalog_name: str | None = None,
-        schema_name: str | None = None,
-        storage_path: str | None = None,
+        path: str | None = None,
+        file_name: str | None = None,
+        table_name: str | None = None,
         **_: Any,
     ) -> PipelineContext:
-        """Reads table metadata from a yaml file using the [`Table`][cloe_nessy.models.table] model.
+        """Reads schema metadata from a yaml file using the [`Schema`][cloe_nessy.models.schema] model.
 
         Args:
             context: The context in which this Action is executed.
-            file_path: The path to the file that defines the table.
-            catalog_name: The name of the catalog for the table.
-            schema_name: The name of the schema for the table.
-            storage_path: The storage path for the table, if applicable. If not
-                provided, the table will be considered a managed table.
+            path: The path to the data contract directory.
+            file_name: The name of the file that defines the schema.
+            table_name: The name of the table for which to retrieve metadata.
 
         Raises:
-            ValueError: If any issues occur while reading the table metadata, such as an invalid table,
-                missing file, missing path, or missing catalog/schema names.
+            ValueError: If any issues occur while reading the schema, such as an invalid schema,
+                missing file, or missing path.
 
         Returns:
             The context after the execution of this Action, containing the table metadata.
         """
-        missing_params = []
-        if not file_path:
-            missing_params.append("file_path")
-        if not catalog_name:
-            missing_params.append("catalog_name")
-        if not schema_name:
-            missing_params.append("schema_name")
+        if not path:
+            raise ValueError("No path provided. Please specify path to schema metadata.")
+        if not file_name:
+            raise ValueError("No file_name provided. Please specify file name.")
+        if not table_name:
+            raise ValueError("No table_name provided. Please specify table name.")
 
-        if missing_params:
-            raise ValueError(
-                f"Missing required parameters: {', '.join(missing_params)}. Please specify all required parameters."
-            )
+        path_obj = pathlib.Path(path)
 
-        final_file_path = Path(file_path) if file_path else Path()
-
-        table, errors = Table.read_instance_from_file(
-            final_file_path,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-        )
+        schema, errors = Schema.read_instance_from_file(path_obj / file_name)
         if errors:
-            raise ValueError(f"Errors while reading table metadata: {errors}")
-        if not table:
-            raise ValueError("No table found in metadata.")
+            raise ValueError(f"Errors while reading schema metadata: {errors}")
+        if not schema:
+            raise ValueError("No schema found in metadata.")
 
-        if not table.storage_path and storage_path:
-            self._console_logger.info(f"Setting storage path for table [ '{table.name}' ] to [ '{storage_path}' ]")
-            table.storage_path = storage_path
-            table.is_external = True
+        table = schema.get_table_by_name(table_name=table_name)
 
-        self._console_logger.info(f"Table [ '{table.name}' ] metadata read successfully from [ '{file_path}' ]")
         return context.from_existing(table_metadata=table)

cloe_nessy/pipeline/actions/transform_join.py

@@ -1,7 +1,5 @@
 from typing import Any
 
-from pyspark.sql import functions as F
-
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 from ..pipeline_step import PipelineStep
@@ -15,74 +13,20 @@ class TransformJoinAction(PipelineAction):
     from [PySpark
     documentation](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 
-    Examples:
-        === "Simple Column Join"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on: id
-                    how: inner
-            ```
-
-        === "Multiple Columns Join"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on: [customer_id, order_date]
-                    how: left
-            ```
-
-        === "Dictionary Join (Different Column Names)"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Transform First Table))
-                    join_on:
-                        customer_id: cust_id
-                        order_date: date
-                    how: inner
-            ```
-
-        === "Complex Join with Literals and Expressions"
-            ```yaml
-            Join Tables:
-                action: TRANSFORM_JOIN
-                options:
-                    joined_data: ((step:Load Conditions Table))
-                    join_condition: |
-                        left.material = right.material
-                        AND right.sales_org = '10'
-                        AND right.distr_chan = '10'
-                        AND right.knart = 'ZUVP'
-                        AND right.lovmkond <> 'X'
-                        AND right.sales_unit = 'ST'
-                        AND left.calday BETWEEN
-                            to_date(right.date_from, 'yyyyMMdd') AND
-                            to_date(right.date_to, 'yyyyMMdd')
-                    how: left
-            ```
+    Example:
+        ```yaml
+        Join Tables:
+            action: TRANSFORM_JOIN
+            options:
+                joined_data: ((step:Transform First Table))
+                join_on: id
+                how: anti
+        ```
 
         !!! note "Referencing a DataFrame from another step"
             The `joined_data` parameter is a reference to the DataFrame from another step.
             The DataFrame is accessed using the `result` attribute of the PipelineStep. The syntax
             for referencing the DataFrame is `((step:Step Name))`, mind the double parentheses.
-
-        !!! tip "Dictionary Join Syntax"
-            When using a dictionary for `join_on`, the keys represent columns
-            from the DataFrame in context and the values represent columns from
-            the DataFrame in `joined_data`. This is useful when joining tables
-            with different column names for the same logical entity.
-
-        !!! tip "Complex Join Conditions"
-            Use `join_condition` instead of `join_on` for complex joins with literals,
-            expressions, and multiple conditions. Reference columns using `left.column_name`
-            for the main DataFrame and `right.column_name` for the joined DataFrame.
-            Supports all PySpark functions and operators.
     """
 
     name: str = "TRANSFORM_JOIN"
@@ -93,7 +37,6 @@ class TransformJoinAction(PipelineAction):
         *,
         joined_data: PipelineStep | None = None,
         join_on: list[str] | str | dict[str, str] | None = None,
-        join_condition: str | None = None,
         how: str = "inner",
         **_: Any,
     ) -> PipelineContext:
@@ -106,17 +49,13 @@ class TransformJoinAction(PipelineAction):
             join_on: A string for the join column
                 name, a list of column names, or a dictionary mapping columns from the
                 left DataFrame to the right DataFrame. This defines the condition for the
-                join operation. Mutually exclusive with join_condition.
-            join_condition: A string containing a complex join expression with literals,
-                functions, and multiple conditions. Use 'left.' and 'right.' prefixes
-                to reference columns from respective DataFrames. Mutually exclusive with join_on.
+                join operation.
             how: The type of join to perform. Must be one of: inner, cross, outer,
                 full, fullouter, left, leftouter, right, rightouter, semi, anti, etc.
 
         Raises:
             ValueError: If no joined_data is provided.
-            ValueError: If neither join_on nor join_condition is provided.
-            ValueError: If both join_on and join_condition are provided.
+            ValueError: If no join_on is provided.
             ValueError: If the data from context is None.
             ValueError: If the data from the joined_data is None.
 
@@ -125,12 +64,8 @@ class TransformJoinAction(PipelineAction):
         """
         if joined_data is None or joined_data.result is None or joined_data.result.data is None:
             raise ValueError("No joined_data provided.")
-
-        if not join_on and not join_condition:
-            raise ValueError("Either join_on or join_condition must be provided.")
-
-        if join_on and join_condition:
-            raise ValueError("Cannot specify both join_on and join_condition. Use one or the other.")
+        if not join_on:
+            raise ValueError("No join_on provided.")
 
         if context.data is None:
             raise ValueError("Data from the context is required for the operation.")
@@ -138,25 +73,16 @@ class TransformJoinAction(PipelineAction):
         df_right = joined_data.result.data.alias("right")  # type: ignore
         df_left = context.data.alias("left")  # type: ignore
 
-        if join_condition:
-            try:
-                condition = F.expr(join_condition)
-            except Exception as e:
-                # this will not raise an error in most cases, because the evaluation of the expression is lazy
-                raise ValueError(f"Failed to parse join condition '{join_condition}': {str(e)}") from e
-            df = df_left.join(df_right, on=condition, how=how)  # type: ignore
-
-        if join_on:
-            if isinstance(join_on, str):
-                join_condition_list = [join_on]
-            elif isinstance(join_on, list):
-                join_condition_list = join_on
-            else:
-                join_condition_list = [
-                    df_left[left_column] == df_right[right_column]  # type: ignore
-                    for left_column, right_column in join_on.items()
-                ]
-
-            df = df_left.join(df_right, on=join_condition_list, how=how)  # type: ignore
+        if isinstance(join_on, str):
+            join_condition = [join_on]
+        elif isinstance(join_on, list):
+            join_condition = join_on
+        else:
+            join_condition = [
+                df_left[left_column] == df_right[right_column]  # type: ignore
+                for left_column, right_column in join_on.items()
+            ]
+
+        df = df_left.join(df_right, on=join_condition, how=how)  # type: ignore
 
         return context.from_existing(data=df)  # type: ignore

cloe_nessy/pipeline/actions/transform_union.py

@@ -22,8 +22,8 @@ class TransformUnionAction(PipelineAction):
             action: TRANSFORM_UNION
             options:
                 union_data:
-                    - ((step:Filter First Table))
-                    - ((step:SQL Transform Second Table))
+                    - ((step: Filter First Table))
+                    - ((step: SQL Transform Second Table))
         ```
         !!! note "Referencing a DataFrame from another step"
             The `union_data` parameter is a reference to the DataFrame from another step.

cloe_nessy/pipeline/actions/write_catalog_table.py

@@ -1,7 +1,6 @@
 from typing import Any
 
 from ...integration.writer import CatalogWriter
-from ...object_manager import TableManager
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
@@ -9,31 +8,17 @@ from ..pipeline_context import PipelineContext
 class WriteCatalogTableAction(PipelineAction):
     """Writes a DataFrame to a specified catalog table using [CatalogWriter][cloe_nessy.integration.writer.CatalogWriter].
 
-    Examples:
-        === "Batch Write"
-            ```yaml
-            Write Table to Catalog:
-                action: WRITE_CATALOG_TABLE
+    Example:
+        ```yaml
+        Write Table to Catalog:
+            action: WRITE_CATALOG_TABLE
+            options:
+                table_identifier: my_catalog.business_schema.sales_table
+                mode: append
+                partition_by: day
                 options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    mode: append
-                    partition_by: day
-                    options:
-                        mergeSchema: true
-            ```
-        === "Streaming Write"
-            ```yaml
-            Write Table to Catalog Stream:
-                action: WRITE_CATALOG_TABLE
-                options:
-                    table_identifier: my_catalog.business_schema.sales_table
-                    mode: append
-                    checkpoint_location: /path/to/checkpoint
-                    trigger_dict:
-                        processingTime: 10 seconds
-                    options:
-                        mergeSchema: true
-            ```
+                    mergeSchema: true
+        ```
     """
 
     name: str = "WRITE_CATALOG_TABLE"
@@ -46,9 +31,6 @@ class WriteCatalogTableAction(PipelineAction):
         mode: str = "append",
         partition_by: str | list[str] | None = None,
         options: dict[str, str] | None = None,
-        checkpoint_location: str | None = None,
-        trigger_dict: dict | None = None,
-        await_termination: bool = False,
         **_: Any,
     ) -> PipelineContext:
         """Writes a DataFrame to a specified catalog table.
@@ -61,11 +43,7 @@ class WriteCatalogTableAction(PipelineAction):
             mode: The write mode. One of 'append', 'overwrite', 'error',
                 'errorifexists', or 'ignore'.
             partition_by: Names of the partitioning columns.
-            checkpoint_location: Location for checkpointing.
-            trigger_dict: A dictionary specifying the trigger configuration for the streaming query.
-            await_termination: If True, the function will wait for the streaming
-                query to finish before returning.
-            options: Additional options for the DataFrame write operation.
+            options: PySpark options for the DataFrame.saveAsTable operation (e.g. mergeSchema:true).
 
         Raises:
             ValueError: If the table name is not specified or cannot be inferred from
@@ -76,15 +54,8 @@ class WriteCatalogTableAction(PipelineAction):
         """
         if not options:
             options = dict()
-        streaming = context.runtime_info and context.runtime_info.get("streaming")
-        if streaming and not checkpoint_location:
-            raise ValueError("Checkpoint location must be specified for streaming writes.")
         if partition_by is None:
-            if (
-                context.table_metadata is not None
-                and hasattr(context.table_metadata, "partition_by")
-                and not context.table_metadata.liquid_clustering
-            ):
+            if hasattr(context.table_metadata, "partition_by"):
                 partition_by = context.table_metadata.partition_by  # type: ignore
 
         if (table_metadata := context.table_metadata) and table_identifier is None:
@@ -92,28 +63,12 @@ class WriteCatalogTableAction(PipelineAction):
         if table_identifier is None:
             raise ValueError("Table name must be specified or a valid Table object with identifier must be set.")
 
-        if table_metadata:
-            manager = TableManager()
-            manager.create_table(table=table_metadata, ignore_if_exists=True, replace=False)
-
         writer = CatalogWriter()
-
-        if streaming:
-            writer.write_stream(
-                df=context.data,  # type: ignore
-                table_identifier=table_identifier,
-                checkpoint_location=checkpoint_location,
-                trigger_dict=trigger_dict,
-                options=options,
-                mode=mode,
-                await_termination=await_termination,
-            )
-        else:
-            writer.write(
-                df=context.data,  # type: ignore
-                table_identifier=table_identifier,
-                mode=mode,
-                partition_by=partition_by,
-                options=options,
-            )
+        writer.write_table(
+            df=context.data,  # type: ignore
+            table_identifier=table_identifier,
+            mode=mode,
+            partition_by=partition_by,
+            options=options,
+        )
         return context.from_existing()

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -98,7 +98,6 @@ class WriteDeltaMergeAction(PipelineAction):
 
         delta_merge_writer.write(
             table_identifier=context.table_metadata.identifier,
-            table=context.table_metadata,
             storage_path=str(context.table_metadata.storage_path),
             data_frame=context.data,
             key_columns=key_columns,

cloe_nessy/session/session_manager.py

@@ -201,6 +201,16 @@ class SessionManager(LoggerMixin):
         except ImportError:
             logger.debug("dbruntime.dbutils not available")
 
+        logger.debug("Checking for Fabric UI...")
+        try:
+            from notebookutils import mssparkutils  # type: ignore # noqa: F401
+
+            logger.debug("✓ Detected FABRIC_UI via notebookutils")
+            cls._env = cls.Environment.FABRIC_UI
+            return cls._env
+        except ImportError:
+            logger.debug("notebookutils not available")
+
         logger.debug("Checking for Databricks Connect...")
         try:
             from databricks.sdk.dbutils import RemoteDbUtils  # type: ignore  # noqa: F401
@@ -212,16 +222,6 @@ class SessionManager(LoggerMixin):
         except ImportError:
             logger.debug("RemoteDbUtils not available")
 
-        logger.debug("Checking for Fabric UI...")
-        try:
-            from notebookutils import mssparkutils  # type: ignore # noqa: F401
-
-            logger.debug("✓ Detected FABRIC_UI via notebookutils")
-            cls._env = cls.Environment.FABRIC_UI
-            return cls._env
-        except ImportError:
-            logger.debug("notebookutils not available")
-
         logger.error("No environment could be detected")
         raise RuntimeError(
             "Cannot detect environment. This usually means you're not in a recognized Spark environment. "

{cloe_nessy-0.3.16.5b0.dist-info → cloe_nessy-0.3.16.6.dist-info}/METADATA

@@ -1,37 +1,36 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.16.5b0
+Version: 0.3.16.6
 Summary: Your friendly datalake monster.
-Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT
+Project-URL: homepage, https://initions.com/
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
-Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
+Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: <3.13,>=3.11
-Requires-Dist: azure-identity<2.0.0,>=1.19.0
-Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
-Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
-Requires-Dist: delta-spark>=3.3.2
-Requires-Dist: fsspec<2025.7.1,>=2025.7.0
-Requires-Dist: httpx<1.0.0,>=0.27.2
-Requires-Dist: jinja2<4.0.0,>=3.1.4
-Requires-Dist: matplotlib<4.0.0,>=3.9.2
-Requires-Dist: networkx<4.0,>=3.3
-Requires-Dist: openpyxl<4.0.0,>=3.1.5
-Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
-Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
+Description-Content-Type: text/markdown
 Requires-Dist: pydantic<3.0.0,>=2.7.2
 Requires-Dist: pyyaml<7.0.0,>=6.0.1
-Requires-Dist: requests<3.0.0,>=2.32.3
-Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
 Requires-Dist: types-pyyaml<7.0.0.0,>=6.0.12.20240311
+Requires-Dist: jinja2<4.0.0,>=3.1.4
+Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
+Requires-Dist: openpyxl<4.0.0,>=3.1.5
+Requires-Dist: requests<3.0.0,>=2.32.3
 Requires-Dist: types-requests<3.0.0.0,>=2.32.0.20240712
-Description-Content-Type: text/markdown
+Requires-Dist: pandas-stubs<3.0.0.0,>=2.2.2.240807
+Requires-Dist: azure-identity<2.0.0,>=1.19.0
+Requires-Dist: httpx<1.0.0,>=0.27.2
+Requires-Dist: databricks-sdk<1.0.0,>=0.36.0
+Requires-Dist: networkx<4.0,>=3.3
+Requires-Dist: matplotlib<4.0.0,>=3.9.2
+Requires-Dist: types-networkx<4.0.0.0,>=3.2.1.20240820
+Requires-Dist: fsspec<2025.7.1,>=2025.7.0
+Requires-Dist: cloe-logging[databricks,log-analytics]<0.4,>=0.3.8
 
 # cloe-nessy
 

{cloe_nessy-0.3.16.5b0.dist-info → cloe_nessy-0.3.16.6.dist-info}/RECORD

@@ -19,18 +19,18 @@ cloe_nessy/file_utilities/strategies/utils_strategy.py,sha256=urayKfOUpSaXKgTs1K
 cloe_nessy/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/integration/reader/__init__.py,sha256=J5vlORqHLBpHEvzIwfIjzN5xEdOat-8jlmdLcGj8nsA,239
 cloe_nessy/integration/reader/api_reader.py,sha256=3Mf-txOTJ1dXCzdNtRTLC8UKftKms4NxOoLVgzcc2eo,5691
-cloe_nessy/integration/reader/catalog_reader.py,sha256=Khw6b5mc0mtya2bT7xq8u0BUg0Mv2qdtAsbMY14pOpg,3741
+cloe_nessy/integration/reader/catalog_reader.py,sha256=lwDeWBVXfFh75XknPawetL9ZBtqS-Oss5rNzbrEeIQg,2070
 cloe_nessy/integration/reader/excel_reader.py,sha256=8KCqKBYFE6RGCiahJimQOAtbYZzaUzlnoslW9yca5P8,8035
 cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
 cloe_nessy/integration/reader/file_reader.py,sha256=3DcZhyyL-Cf_R7Px1UDHJwpO8Un31dWey2Q-f4DtWfY,6879
 cloe_nessy/integration/reader/reader.py,sha256=e2KVPePQme8SBQJEbL-3zpGasOgTiEvKFTslow2wGPw,1034
 cloe_nessy/integration/writer/__init__.py,sha256=3yzCAGiWZdQWtsbzlTih01sxVTJV2DDYwvl34lEAUlE,243
-cloe_nessy/integration/writer/catalog_writer.py,sha256=Z26FOL3D9KK6I7Y3rgl4c88rToKZnVXlELTYH2xQsHY,5289
+cloe_nessy/integration/writer/catalog_writer.py,sha256=Gb-hMdADgO_uUJ7mZPHBYyNme2qXsdFFnzwo7GcShHM,2192
 cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70To4L6Q182pXx2HRM,5454
 cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
 cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
 cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=TbpW-j87_H9dcUza34uR6VWslJez406y3_5N1ip0SnM,4740
-cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=zhqPIPfAJTzSLFgBUCwFesUW7CcF1zCPRU-N_8yYjok,10172
+cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=no2GOLqMAJd0fEy2mqMevMj_CvutcJPRmXJC2tD4icA,10112
 cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=kiacqQ2FYQSzakJqZ9-ZHH3os4X7--QuER_2xx9y21k,971
 cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=upUtDZMzwYFU0kzmkelVgkpFToXkrypcR3h_jvGjz14,8596
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
@@ -38,10 +38,10 @@ cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZum
 cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
 cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
 cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
-cloe_nessy/models/column.py,sha256=IQarCPM1J-g25AHAAHy4Z9MDN5hyyTwJJQGC5yZYcas,2021
+cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
 cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
 cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
-cloe_nessy/models/schema.py,sha256=cNSrH7K4hLRrkg1E6fW6DUIBMZdR2A5B21POj5iQ4GA,3429
+cloe_nessy/models/schema.py,sha256=yUrjjEhAH5zbCymE67Az_jPnVB8hGO-_UNfqzeZCD_Y,3376
 cloe_nessy/models/table.py,sha256=O9vcJ1XBIb6kA-NAI3SNpB5b7MGDo3p4wMJdonPaBfA,12076
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
 cloe_nessy/models/volume.py,sha256=51BE06FrL1Wv6zblFwJ_HTiR6WQqH7pSmrdH90rqwLg,2444
@@ -50,10 +50,10 @@ cloe_nessy/models/adapter/unity_catalog_adapter.py,sha256=a-14Ys-AevVYQd0xeJU1sy
 cloe_nessy/models/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/models/mixins/read_instance_mixin.py,sha256=j5Y4aNWOh1jlskEaxNooZFJgPyxRmik00gAVLJnAaRs,4507
 cloe_nessy/models/mixins/template_loader_mixin.py,sha256=5MXhEGBFlq3dwZvINEyBowSlipNnVun2H_TmhI_fsS4,549
-cloe_nessy/models/templates/create_table.sql.j2,sha256=71JpUyUZ_ZYO2M0tfIrTXHR7JycypAGsELt2-2d3oO0,2479
+cloe_nessy/models/templates/create_table.sql.j2,sha256=QWbiTXwmGaIlZUAIGL4pAlHkDbP9mq1vGAkdKCPOqm4,1669
 cloe_nessy/models/templates/create_volume.sql.j2,sha256=XIUf1cHcvAxcGTyhzUiv4xpQ1cfDw_ra3_FKmOuLoBs,289
 cloe_nessy/object_manager/__init__.py,sha256=3sle0vNpPwBOkycxA3XVS9m4XZf5LD3Qd4NGxdqcHno,186
-cloe_nessy/object_manager/table_manager.py,sha256=suHx56TYXagaJ2dVkvTP7vwSI4xgTqXNkHYBbYh2pd4,13913
+cloe_nessy/object_manager/table_manager.py,sha256=m6u_KFYCPoqq1hagwt3s7eQopjV2oOJNlmXDVAfku-k,12703
 cloe_nessy/object_manager/volume_manager.py,sha256=6epd3KXzcNH04EvaKubAfLsaUm9qBMeT3KNvMK04gGs,2727
 cloe_nessy/pipeline/__init__.py,sha256=sespmJ5JsgyiFyZiedTiL2kg--zGIX7cjTYsD5vemEg,325
 cloe_nessy/pipeline/pipeline.py,sha256=L4wk3b06LNWRj01nnAkuQpeRrwFTyaV1xTpgYAg4sak,10819
@@ -65,14 +65,13 @@ cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
 cloe_nessy/pipeline/actions/__init__.py,sha256=9gjSQKLGrPcaYaJrTYZde8d4yNrN1SoXN_DDHq5KrvY,2600
 cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
-cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=NV2uTfOASKrbxlC1zLYYP8ODnV52m5eGU5MoU3xGFfE,3799
+cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=oXbqbc6BfR82dSIGclwzWiTN8EVmpFjNIYLKm4qOU50,2754
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
 cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
-cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=i8fQceV63eAqx_x0ANisCkXWfMHyhqsfFHVFH5yP2po,3544
+cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
-cloe_nessy/pipeline/actions/transform_convert_timestamp.py,sha256=je6H-mtNeokU9W_-RCWaRCFvMhk4oQL9s60FVBrl8Po,3090
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
 cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
 cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
@@ -80,22 +79,23 @@ cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1Qeat
 cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=_naWfmPdYAUKjPNeHu5qJAohOL7DHCSYz_kwoeRv3OI,2741
 cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpbsPEJkzea5zFJA6MuyjNpOsFud9o,4045
 cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=heRjBA-Gfu-nmNHOjTYlipEpKY8oNPAHAY40vjJk3aI,8383
-cloe_nessy/pipeline/actions/transform_join.py,sha256=ez1M1wVc9khOZj1swMArJbBKXxEpjenUHrW1wL8H330,7200
+cloe_nessy/pipeline/actions/transform_join.py,sha256=e_tvMk8YJTAWcUK_EmOgNt0s31ICZoMX_MKOTWx4lBY,3645
 cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
 cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
 cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=1OPHTrjcphfyGepcO7ozYfeqfwA18pjlyHpVKUS_AAU,2049
 cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm3BG9BX4_qUDJMbpj1DsKPY046w,4574
-cloe_nessy/pipeline/actions/transform_union.py,sha256=SZtEzh567CIExUj9yMEgshE28h4dXKT7Wr2TDj4zB4k,2718
-cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=idsEypBqnThHqo2ghrpYGhHSzn4tswCxIa9mbZP_xhA,4642
+cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
+cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
 cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPexeK0XNXUfQscCwhXHxbKA,2498
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=gh3oD0ZGjDq0hw56NiRimK4HHCruDofqqdzFFgYLve8,5085
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
 cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
-cloe_nessy/session/session_manager.py,sha256=sgsN_U6c_IjYF1wSjd8Opj9FV1gjAvaqVy4ljFjk8AQ,9710
+cloe_nessy/session/session_manager.py,sha256=VCUPhACeN5armd4D0TqDeH4Ih9nu6XvXSREFqHUwt4s,9710
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.16.5b0.dist-info/METADATA,sha256=k06Hu0tT4c7JkwmN7WmDVTBvZi8ypdB5Agxfty42CtQ,3328
-cloe_nessy-0.3.16.5b0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cloe_nessy-0.3.16.5b0.dist-info/RECORD,,
+cloe_nessy-0.3.16.6.dist-info/METADATA,sha256=YfBuBVqeRWjBTWlj4SQKyUVrc-PX78fK_MnHhO2MQv4,3292
+cloe_nessy-0.3.16.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.16.6.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.16.6.dist-info/RECORD,,

{cloe_nessy-0.3.16.5b0.dist-info → cloe_nessy-0.3.16.6.dist-info}/WHEEL

@@ -1,4 +1,5 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
+

cloe_nessy-0.3.16.6.dist-info/top_level.txt

@@ -0,0 +1 @@
+cloe_nessy

cloe_nessy/pipeline/actions/transform_convert_timestamp.py

@@ -1,87 +0,0 @@
-from typing import Any
-
-from pyspark.errors.exceptions.base import IllegalArgumentException
-from pyspark.sql import functions as F
-
-from ...pipeline import PipelineAction, PipelineContext
-
-
-class TransformConvertTimestampAction(PipelineAction):
-    """This class implements a Transform action for an ETL pipeline.
-
-    This action performs timestamp based conversions.
-
-    Example:
-        ```yaml
-        Convert Timestamp:
-            action: TRANSFORM_CONVERT_TIMESTAMP
-            options:
-                column: my_timestamp_column
-                source_format: unixtime
-                target_format: yyyy-MM-dd HH:mm:ss
-        ```
-    """
-
-    name: str = "TRANSFORM_CONVERT_TIMESTAMP"
-
-    def run(
-        self,
-        context: PipelineContext,
-        *,
-        column: str = "",
-        source_format: str = "",
-        target_format: str = "",
-        **_: Any,
-    ) -> PipelineContext:
-        """Converts a column from a given source format to a new format.
-
-        Args:
-            context: Context in which this Action is executed.
-            column: The column that should be converted.
-            source_format: Initial format type of the column.
-            target_format: Desired format type of the column. This also supports
-            passing a format string like 'yyyy-MM-dd HH:mm:ss'.
-
-        Raises:
-            ValueError: If no column, source_format and target_format are provided.
-            ValueError: If source_format or target_format are not supported.
-
-        Returns:
-            PipelineContext: Context after the execution of this Action.
-        """
-        if not column:
-            raise ValueError("No column provided.")
-        if not source_format:
-            raise ValueError("No source_format provided.")
-        if not target_format:
-            raise ValueError("No target_format provided.")
-        if context.data is None:
-            raise ValueError("Context DataFrame is required.")
-        df = context.data
-
-        match source_format:
-            # convert always to timestamp first
-            case "unixtime":
-                df = df.withColumn(column, F.from_unixtime(F.col(column)))
-            case "unixtime_ms":
-                df = df.withColumn(column, F.to_timestamp(F.col(column) / 1000))
-            case "string":
-                df = df.withColumn(column, F.to_timestamp(F.col(column)))
-            case "timestamp":
-                pass
-            case _:
-                raise ValueError(f"Unknown source_format {source_format}")
-
-        match target_format:
-            # convert from timestamp to desired output format
-            case "timestamp":
-                pass
-            case "unixtime":
-                df = df.withColumn(column, F.to_unix_timestamp(F.col(column)))
-            case _:
-                try:
-                    df = df.withColumn(column, F.date_format(F.col(column), target_format))
-                except IllegalArgumentException as e:
-                    raise ValueError(f"Invalid target_format {target_format}") from e
-
-        return context.from_existing(data=df)