cloe-nessy 0.3.16.6__py3-none-any.whl → 0.3.16.7__py3-none-any.whl

cloe_nessy/models/table.py

@@ -2,7 +2,6 @@ from pathlib import Path
 from typing import Any, Self
 
 import yaml
-import yaml.scanner
 from jinja2 import TemplateNotFound
 from pydantic import (
     Field,
@@ -11,6 +10,8 @@ from pydantic import (
     field_validator,
     model_validator,
 )
+from yaml.parser import ParserError
+from yaml.scanner import ScannerError
 
 from ..logging import LoggerMixin
 from ..utils.file_and_directory_handler import process_path
@@ -225,8 +226,8 @@ class Table(TemplateLoaderMixin, ReadInstancesMixin, LoggerMixin):
                 errors += sub_errors
         except (
             ValidationError,
-            yaml.parser.ParserError,
-            yaml.scanner.ScannerError,
+            ParserError,
+            ScannerError,
         ) as e:
             instance = None
             errors.append(e)

cloe_nessy/pipeline/actions/__init__.py

@@ -10,6 +10,7 @@ from .transform_change_datatype import TransformChangeDatatypeAction
 from .transform_clean_column_names import TransformCleanColumnNamesAction
 from .transform_concat_columns import TransformConcatColumnsAction
 from .transform_decode import TransformDecodeAction
+from .transform_deduplication import TransformDeduplication
 from .transform_distinct import TransformDistinctAction
 from .transform_filter import TransformFilterAction
 from .transform_generic_sql import TransformSqlAction
@@ -46,6 +47,7 @@ __all__ = [
     "TransformCleanColumnNamesAction",
     "TransformConcatColumnsAction",
     "TransformDecodeAction",
+    "TransformDeduplication",
     "TransformDistinctAction",
     "TransformSqlAction",
     "TransformGroupAggregate",

cloe_nessy/pipeline/actions/read_catalog_table.py

@@ -1,8 +1,10 @@
 from typing import Any
 
+from ...integration.delta_loader import DeltaLoadOptions
 from ...integration.reader import CatalogReader
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
+from ..utils import set_delta_load_info
 
 
 class ReadCatalogTableAction(PipelineAction):
@@ -21,6 +23,12 @@ class ReadCatalogTableAction(PipelineAction):
             options:
                 table_identifier: my_catalog.business_schema.sales_table
                 options: <options for the CatalogReader read method>
+            delta_load_options:
+                strategy: CDF
+                delta_load_identifier: my_delta_load_id
+                strategy_options:
+                    deduplication_columns: ["id"]
+                    enable_full_load: true
         ```
     """
 
@@ -32,6 +40,7 @@ class ReadCatalogTableAction(PipelineAction):
         *,
         table_identifier: str | None = None,
         options: dict[str, str] | None = None,
+        delta_load_options: dict[Any, Any] | DeltaLoadOptions | None = None,
         **_: Any,  # define kwargs to match the base class signature
     ) -> PipelineContext:
         """Reads a table from Unity Catalog using a specified table identifier and optional reader configurations.
@@ -45,6 +54,8 @@ class ReadCatalogTableAction(PipelineAction):
             options: A dictionary of options for customizing
                 the [`CatalogReader`][cloe_nessy.integration.reader.catalog_reader]
                 behavior, such as filters or reading modes. Defaults to None.
+            delta_load_options: Options for delta loading, if applicable.
+                Configures the [`DeltaLoader`][cloe_nessy.integration.delta_loader].
 
         Raises:
             ValueError: If neither `table_identifier` nor `table_metadata.identifier` in the `context` is provided.
@@ -53,13 +64,35 @@ class ReadCatalogTableAction(PipelineAction):
         An updated pipeline context containing the data read from the catalog table as a DataFrame.
         """
         if not options:
-            options = dict()
+            options = {}
+
+        if not delta_load_options:
+            delta_load_options = {}
 
         if (table_metadata := context.table_metadata) and table_identifier is None:
             table_identifier = table_metadata.identifier
         if table_identifier is None:
             raise ValueError("Table name must be specified or a valid Table object with identifier must be set.")
 
+        if isinstance(delta_load_options, dict):
+            delta_options_dict = delta_load_options
+            if delta_load_options:
+                delta_load_options = DeltaLoadOptions(**delta_load_options)
+            else:
+                delta_load_options = None
+        else:
+            delta_options_dict = delta_load_options.model_dump() if delta_load_options else {}
+
+        runtime_info = set_delta_load_info(
+            table_identifier=table_identifier,
+            delta_load_options=delta_options_dict,
+            runtime_info=context.runtime_info or {},
+        )
+
         table_reader = CatalogReader()
-        df = table_reader.read(table_identifier=table_identifier, options=options)
-        return context.from_existing(data=df)
+        df = table_reader.read(
+            table_identifier=table_identifier,
+            options=options,
+            delta_load_options=delta_load_options,
+        )
+        return context.from_existing(data=df, runtime_info=runtime_info)

cloe_nessy/pipeline/actions/read_files.py

@@ -1,8 +1,10 @@
 from typing import Any
 
+from ...integration.delta_loader import DeltaLoadOptions
 from ...integration.reader import FileReader
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
+from ..utils import set_delta_load_info
 
 
 class ReadFilesAction(PipelineAction):
@@ -55,6 +57,24 @@ class ReadFilesAction(PipelineAction):
                 Use the `extension` option to specify the extension of the files
                 to read. Additionally, use the `spark_format` option to specify
                 the format with which to read the files.
+
+        === "Read Delta Lake table with delta loading"
+            ```yaml
+            Read Delta Files:
+                action: READ_FILES
+                options:
+                    location: /path/to/delta/table
+                    spark_format: delta
+                delta_load_options:
+                    strategy: CDF
+                    delta_load_identifier: my_delta_files_load
+                    strategy_options:
+                        deduplication_columns: ["id"]
+                        enable_full_load: false
+            ```
+            !!! note "Delta Loading for Files"
+                Use `delta_load_options` when reading Delta Lake tables to enable
+                incremental loading. This works with both CDF and timestamp strategies.
     """
 
     name: str = "READ_FILES"
@@ -70,6 +90,7 @@ class ReadFilesAction(PipelineAction):
         schema: str | None = None,
         add_metadata_column: bool = True,
         options: dict[str, str] | None = None,
+        delta_load_options: dict[Any, Any] | DeltaLoadOptions | None = None,
         **_: Any,
     ) -> PipelineContext:
         """Reads files from a specified location.
@@ -87,6 +108,8 @@ class ReadFilesAction(PipelineAction):
             add_metadata_column: Whether to include the `__metadata` column with
                 file metadata in the DataFrame.
             options: Additional options passed to the reader.
+            delta_load_options: Options for delta loading, if applicable. When provided
+                for Delta format files, enables incremental loading using delta loader strategies.
 
         Raises:
             ValueError: If neither `extension` nor `spark_format` are provided, or if
@@ -105,6 +128,25 @@ class ReadFilesAction(PipelineAction):
         if (metadata := context.table_metadata) and schema is None:
             schema = metadata.schema
 
+        # Convert dict to DeltaLoadOptions if needed
+        if isinstance(delta_load_options, dict):
+            delta_load_options = DeltaLoadOptions(**delta_load_options)
+
+        # Set up runtime info for delta loading
+        runtime_info = context.runtime_info or {}
+        if delta_load_options:
+            # Convert DeltaLoadOptions to dict for runtime info storage
+            delta_options_dict = (
+                delta_load_options.model_dump()
+                if isinstance(delta_load_options, DeltaLoadOptions)
+                else delta_load_options
+            )
+            runtime_info = set_delta_load_info(
+                table_identifier=location,  # Use location as identifier for file-based delta loading
+                delta_load_options=delta_options_dict,
+                runtime_info=runtime_info,
+            )
+
         file_reader = FileReader()
         df = file_reader.read(
             location=location,
@@ -114,11 +156,11 @@ class ReadFilesAction(PipelineAction):
             search_subdirs=search_subdirs,
             options=options,
             add_metadata_column=add_metadata_column,
+            delta_load_options=delta_load_options,
         )
 
-        runtime_info = context.runtime_info
-
-        if add_metadata_column:
+        # Only process metadata column if it exists and wasn't using delta loading
+        if add_metadata_column and "__metadata" in df.columns:
             read_files_list = [x.file_path for x in df.select("__metadata.file_path").drop_duplicates().collect()]
             if runtime_info is None:
                 runtime_info = {"read_files": read_files_list}

cloe_nessy/pipeline/actions/transform_deduplication.py

@@ -1,11 +1,10 @@
-import random
-import string
 from typing import Any
 
 import pyspark.sql.functions as F
 import pyspark.sql.types as T
 from pyspark.sql import Window
 
+from ...utils.column_names import generate_unique_column_name
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
 
@@ -105,18 +104,14 @@ class TransformDeduplication(PipelineAction):
         else:
             order_by_list = [F.col(col_name).asc() for col_name in order_by_columns]
 
-        # create the window specification
-        window_specification = Window.partitionBy(key_columns).orderBy(order_by_list)
-
-        # generate a column name that is not in the input dataframe
-        def generate_random_string(length):
-            return "".join(random.choice(string.ascii_uppercase) for _ in range(length))
+        window_specification = (
+            Window.partitionBy(key_columns)
+            .orderBy(order_by_list)
+            .rowsBetween(Window.unboundedPreceding, Window.currentRow)
+        )
 
-        row_number_col_name = generate_random_string(20)
-        while row_number_col_name in context.data.columns:
-            row_number_col_name = generate_random_string(20)
+        row_number_col_name = generate_unique_column_name(existing_columns=set(context.data.columns), prefix="row_num")
 
-        # drop the duplicates
         df = (
             context.data.withColumn(row_number_col_name, F.row_number().over(window_specification))
             .filter(F.col(row_number_col_name) == 1)

cloe_nessy/pipeline/actions/transform_hash_columns.py

@@ -132,13 +132,13 @@ class TransformHashColumnsAction(PipelineAction):
             action: TRANSFORM_HASH_COLUMNS
             options:
                 hash_config:
-                - hashed_column1:
-                    columns: ["column1", "column2"]
-                    algorithm: "sha2"
-                    bits: 224
-                - hashed_column2:
-                    columns: ["column1"]
-                    algorithm: "crc32"
+                    hashed_column1:
+                        columns: ["column1", "column2"]
+                        algorithm: "sha2"
+                        bits: 224
+                    hashed_column2:
+                        columns: ["column1"]
+                        algorithm: "crc32"
         ```
 
     Given a DataFrame `df` with the following structure:

cloe_nessy/pipeline/actions/write_catalog_table.py

@@ -1,5 +1,6 @@
 from typing import Any
 
+from ...integration.delta_loader import consume_delta_load
 from ...integration.writer import CatalogWriter
 from ..pipeline_action import PipelineAction
 from ..pipeline_context import PipelineContext
@@ -63,6 +64,10 @@ class WriteCatalogTableAction(PipelineAction):
         if table_identifier is None:
             raise ValueError("Table name must be specified or a valid Table object with identifier must be set.")
 
+        runtime_info = getattr(context, "runtime_info", None)
+        if runtime_info and runtime_info.get("is_delta_load"):
+            consume_delta_load(runtime_info)
+
         writer = CatalogWriter()
         writer.write_table(
             df=context.data,  # type: ignore

cloe_nessy/pipeline/actions/write_delta_append.py

@@ -1,5 +1,6 @@
 from typing import Any
 
+from ...integration.delta_loader import consume_delta_load
 from ...integration.writer import DeltaAppendWriter
 from ...models.adapter import UnityCatalogAdapter
 from ...pipeline import PipelineAction, PipelineContext
@@ -10,6 +11,15 @@ class WriteDeltaAppendAction(PipelineAction):
 
     The WriteDeltaAppendAction appends a Dataframe to Delta Table.
 
+    Example:
+        ```yaml
+        Write Delta Append:
+            action: WRITE_DELTA_APPEND
+            options:
+                table_identifier: my_catalog.my_schema.my_table
+                ignore_empty_df: false
+        ```
+
     Returns:
         None.
     """
@@ -66,4 +76,9 @@ class WriteDeltaAppendAction(PipelineAction):
             ignore_empty_df=ignore_empty_df,
             options=options,
         )
+
+        runtime_info = getattr(context, "runtime_info", None)
+        if runtime_info and runtime_info.get("is_delta_load"):
+            consume_delta_load(runtime_info)
+
         return context.from_existing()

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -1,5 +1,6 @@
 from typing import Any
 
+from ...integration.delta_loader import consume_delta_load
 from ...integration.writer import DeltaMergeWriter
 from ...models.adapter import UnityCatalogAdapter
 from ...pipeline import PipelineAction, PipelineContext
@@ -10,6 +11,24 @@ class WriteDeltaMergeAction(PipelineAction):
 
     The MergeIntoDeltaAction merges a Dataframe to Delta Table.
 
+    Example:
+        ```yaml
+        Write Delta Merge:
+            action: WRITE_DELTA_MERGE
+            options:
+                table_identifier: my_catalog.my_schema.my_table
+                key_columns:
+                    - id
+                    - customer_id
+                cols_to_update:
+                    - name
+                    - email
+                    - updated_at
+                when_matched_update: true
+                when_not_matched_insert: true
+                use_partition_pruning: true
+        ```
+
     Returns:
         None.
     """
@@ -112,6 +131,10 @@ class WriteDeltaMergeAction(PipelineAction):
             ignore_empty_df=ignore_empty_df,
         )
 
+        runtime_info = getattr(context, "runtime_info", None)
+        if runtime_info and runtime_info.get("is_delta_load"):
+            consume_delta_load(runtime_info)
+
         if refresh_table:
             delta_merge_writer.table_manager.refresh_table(table_identifier=context.table_metadata.identifier)
 

cloe_nessy/pipeline/actions/write_file.py

@@ -1,5 +1,6 @@
 from typing import Any
 
+from ...integration.delta_loader import consume_delta_load
 from ...integration.writer import FileWriter
 from ...pipeline import PipelineAction, PipelineContext
 
@@ -21,7 +22,7 @@ class WriteFileAction(PipelineAction):
                 mode: "append"
                 is_stream: False
                 options:
-                    mergeSchema: "true"
+                    mergeSchema: true
         ```
     """
 
@@ -91,4 +92,8 @@ class WriteFileAction(PipelineAction):
                 options=options,
             )
 
+        runtime_info = getattr(context, "runtime_info", None)
+        if runtime_info and runtime_info.get("is_delta_load"):
+            consume_delta_load(runtime_info)
+
         return context.from_existing()

cloe_nessy/pipeline/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Pipeline utility modules."""
+
+from .delta_load_utils import set_delta_load_info
+
+__all__ = ["set_delta_load_info"]

cloe_nessy/pipeline/utils/delta_load_utils.py

@@ -0,0 +1,36 @@
+"""Utilities for managing delta load information in pipeline runtime context."""
+
+from typing import Any
+
+
+def set_delta_load_info(
+    table_identifier: str,
+    delta_load_options: dict[str, Any],
+    runtime_info: dict[str, Any],
+) -> dict[str, Any]:
+    """Update the runtime information dictionary with delta load options for a specific table.
+
+    If delta load options are provided, this function marks the runtime as a delta load and
+    stores the options under the given table identifier within the 'delta_load_options' key
+    of the runtime_info dictionary.
+
+    The method uses `setdefault("delta_load_options", {})` to ensure that the 'delta_load_options'
+    key exists in the runtime_info dictionary. If the key is not present, it initializes it with
+    an empty dictionary. This prevents overwriting existing delta load options and allows
+    multiple tables' options to be stored without losing previous entries.
+
+    Args:
+        table_identifier: The identifier for the table (can be table name or file path).
+        delta_load_options: Options specific to the delta load for the table.
+        runtime_info: The runtime information dictionary to update.
+
+    Returns:
+        The updated runtime information dictionary with delta load details.
+    """
+    if not delta_load_options:
+        return runtime_info
+
+    runtime_info["is_delta_load"] = True
+    runtime_info.setdefault("delta_load_options", {})[table_identifier] = delta_load_options
+
+    return runtime_info

cloe_nessy/utils/column_names.py

@@ -0,0 +1,9 @@
+import uuid
+
+
+def generate_unique_column_name(existing_columns: set[str], prefix: str = "temp_col") -> str:
+    """Generate a unique column name that doesn't conflict with existing columns."""
+    base_name = f"{prefix}_{uuid.uuid4().hex[:8]}"
+    while base_name in existing_columns:
+        base_name = f"{prefix}_{uuid.uuid4().hex[:8]}"
+    return base_name

{cloe_nessy-0.3.16.6.dist-info → cloe_nessy-0.3.16.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 0.3.16.6
+Version: 0.3.16.7
 Summary: Your friendly datalake monster.
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>
 License: MIT

{cloe_nessy-0.3.16.6.dist-info → cloe_nessy-0.3.16.7.dist-info}/RECORD

@@ -17,12 +17,20 @@ cloe_nessy/file_utilities/strategies/local_strategy.py,sha256=6OcEjzLvRTBT8FKXhk
 cloe_nessy/file_utilities/strategies/onelake_strategy.py,sha256=RnQjWtWIFzFj-zPqzyZaPYIjtjXkgP-K7-VA8GhkNmg,1980
 cloe_nessy/file_utilities/strategies/utils_strategy.py,sha256=urayKfOUpSaXKgTs1KVK0TS7FWVrJ3k4OLKh35sCxAU,3194
 cloe_nessy/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cloe_nessy/integration/delta_loader/__init__.py,sha256=ZdBDde1uPtTCL_KAhilVmtVmmGvH5dHb05QsOozkteE,438
+cloe_nessy/integration/delta_loader/delta_load_options.py,sha256=bbPGhC0n8L6CmcmV91Xqq6fWRimxlUHUkr22uVqG0g4,1363
+cloe_nessy/integration/delta_loader/delta_loader.py,sha256=D5oOvVLWRwl0z0iQScXVOapErAl6Z5Kt3qXedchgq0s,6878
+cloe_nessy/integration/delta_loader/delta_loader_factory.py,sha256=vB1cL6-Nc3SkLH1xtazMbMF1MnNYq8-g3GHZzRE3QmE,2251
+cloe_nessy/integration/delta_loader/delta_loader_metadata_table.py,sha256=G_EWUY76ZlbsPZB9LCGlOLVezk7DK6peYXEgt7-sTQE,1683
+cloe_nessy/integration/delta_loader/strategies/__init__.py,sha256=1o5fRWenL5KnUg1hf7kmTuTpG9pbMxchiQTub52Qvwo,255
+cloe_nessy/integration/delta_loader/strategies/delta_cdf_loader.py,sha256=FOOZqtMwp8_LoyG2ab2N19a074CFa2ArCEvNkl7wRWM,16682
+cloe_nessy/integration/delta_loader/strategies/delta_timestamp_loader.py,sha256=YYFH0DkdRPvITUc1JMgkmgIHjwDyZDCjqvEk2qhBMfE,6185
 cloe_nessy/integration/reader/__init__.py,sha256=J5vlORqHLBpHEvzIwfIjzN5xEdOat-8jlmdLcGj8nsA,239
 cloe_nessy/integration/reader/api_reader.py,sha256=3Mf-txOTJ1dXCzdNtRTLC8UKftKms4NxOoLVgzcc2eo,5691
-cloe_nessy/integration/reader/catalog_reader.py,sha256=lwDeWBVXfFh75XknPawetL9ZBtqS-Oss5rNzbrEeIQg,2070
+cloe_nessy/integration/reader/catalog_reader.py,sha256=7jFuqIPpuz03opULh2I0TCLPfW6AqkxjaW2kCc0oM1g,3292
 cloe_nessy/integration/reader/excel_reader.py,sha256=8KCqKBYFE6RGCiahJimQOAtbYZzaUzlnoslW9yca5P8,8035
 cloe_nessy/integration/reader/exceptions.py,sha256=_A9jFpe_RIDZCGY76qzjic9bsshxns6yXPSl141dq1c,203
-cloe_nessy/integration/reader/file_reader.py,sha256=3DcZhyyL-Cf_R7Px1UDHJwpO8Un31dWey2Q-f4DtWfY,6879
+cloe_nessy/integration/reader/file_reader.py,sha256=Za_DZKUq1vATp8kIS8uY9IDHiaReZO0k80rrPHAhi5A,8132
 cloe_nessy/integration/reader/reader.py,sha256=e2KVPePQme8SBQJEbL-3zpGasOgTiEvKFTslow2wGPw,1034
 cloe_nessy/integration/writer/__init__.py,sha256=3yzCAGiWZdQWtsbzlTih01sxVTJV2DDYwvl34lEAUlE,243
 cloe_nessy/integration/writer/catalog_writer.py,sha256=Gb-hMdADgO_uUJ7mZPHBYyNme2qXsdFFnzwo7GcShHM,2192
@@ -31,18 +39,18 @@ cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuO
 cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
 cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=TbpW-j87_H9dcUza34uR6VWslJez406y3_5N1ip0SnM,4740
 cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=no2GOLqMAJd0fEy2mqMevMj_CvutcJPRmXJC2tD4icA,10112
-cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=kiacqQ2FYQSzakJqZ9-ZHH3os4X7--QuER_2xx9y21k,971
+cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=m4YFY9_WgaOcnpBviVt3Km-w3wf3NF25wPS-n0NBGcE,970
 cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=upUtDZMzwYFU0kzmkelVgkpFToXkrypcR3h_jvGjz14,8596
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
 cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
-cloe_nessy/logging/logger_mixin.py,sha256=9iy7BF6drYme-f7Rrt_imbVBRgVqQ89xjcP1X5aMtfY,7467
+cloe_nessy/logging/logger_mixin.py,sha256=H8MyMEyb_kEDP0Ow5QStAFLuOkTIeUnneGaj916fKlU,7443
 cloe_nessy/models/__init__.py,sha256=-FmWEJ1Oq1njSopjc0R7GmT64mLSmALkm8PkHNzy9Y8,327
 cloe_nessy/models/catalog.py,sha256=ayC1sMp4cNLAZtu0ICVV3Us6-o4hn8U9tpzzvxC9RAs,177
-cloe_nessy/models/column.py,sha256=53fBwRnino72XKACsHZpN9QfCBqqSXyKLHZlM0huumg,1988
+cloe_nessy/models/column.py,sha256=t-MX9GMs7l5W0APvsUxiE1TI9SWkKdFKblmz24s4IHY,1995
 cloe_nessy/models/constraint.py,sha256=hsFlhn4n928z81O3dl3v5bMetewPWzMjkJK3_4kASSM,178
 cloe_nessy/models/foreign_key.py,sha256=DwRVHs9sShqqPV-NL7ow_3AmPPWX0Od26yZn_I565pU,1001
 cloe_nessy/models/schema.py,sha256=yUrjjEhAH5zbCymE67Az_jPnVB8hGO-_UNfqzeZCD_Y,3376
-cloe_nessy/models/table.py,sha256=O9vcJ1XBIb6kA-NAI3SNpB5b7MGDo3p4wMJdonPaBfA,12076
+cloe_nessy/models/table.py,sha256=3AUBUKLJv1x-xN9KYc5Ndjf-lAlT83rUYdhRKy8wFU4,12105
 cloe_nessy/models/types.py,sha256=XRbuJGdTNa6aXyE3IAzs_J9gVjbfkzMDLfGl-k6jI_4,223
 cloe_nessy/models/volume.py,sha256=51BE06FrL1Wv6zblFwJ_HTiR6WQqH7pSmrdH90rqwLg,2444
 cloe_nessy/models/adapter/__init__.py,sha256=m36W_mqwB3dCYnCIt0fLOSHS4E1VU8FRGoaum4Gf95o,90
@@ -63,39 +71,42 @@ cloe_nessy/pipeline/pipeline_context.py,sha256=csElDc6BsynDUtRXgQOSCH7ONc_b-ag0Y
 cloe_nessy/pipeline/pipeline_parsing_service.py,sha256=c_nAsgw81QYBM9AFiTxGgqRhNXABkDKplbeoCJPtbpE,6434
 cloe_nessy/pipeline/pipeline_plotting_service.py,sha256=goMQj73FzUVchKn5c2SsPcWR6fr7DtVkVrcQfJsKCq4,13111
 cloe_nessy/pipeline/pipeline_step.py,sha256=UlnmpS6gm_dZ7m9dD1mZvye7mvUF_DA7HjOZo0oGYDU,1977
-cloe_nessy/pipeline/actions/__init__.py,sha256=9gjSQKLGrPcaYaJrTYZde8d4yNrN1SoXN_DDHq5KrvY,2600
+cloe_nessy/pipeline/actions/__init__.py,sha256=RAGwu3Xzt2JJc0AveZ_hVi3SxTIdehAG-JoQe9JPMNA,2690
 cloe_nessy/pipeline/actions/read_api.py,sha256=RBv5XeHtjTXuCP09Fqo6JNx6iIhQQI-nuAHCuSaGs2s,7778
-cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=oXbqbc6BfR82dSIGclwzWiTN8EVmpFjNIYLKm4qOU50,2754
+cloe_nessy/pipeline/actions/read_catalog_table.py,sha256=TBlJaXJAQwLtwvh7dXsX9ebNN3rS6En6951MnT8xGG8,4101
 cloe_nessy/pipeline/actions/read_excel.py,sha256=Mhl3r_2Hqk2XN7Fl5WqqAyE4JdnwSiivbhWMglyBtkE,7961
-cloe_nessy/pipeline/actions/read_files.py,sha256=N9bFgtG1tovhp2JayxE5YiN9PiO2lgG2-6h_Y6tD2eU,5220
+cloe_nessy/pipeline/actions/read_files.py,sha256=hRcM7wG35vxxLVajW3SK5euHW02qxiXCYSkIl11xiQ0,7308
 cloe_nessy/pipeline/actions/read_metadata_yaml.py,sha256=3ZDy9qiDYtM1oDQzHPC23hLOvHjhdk5zg1wVHE60m9k,2295
 cloe_nessy/pipeline/actions/transform_change_datatype.py,sha256=24Tn6R3TvUkWCh8V6naLdyNbCbqvyPOOoer-hy_Ebq4,2077
 cloe_nessy/pipeline/actions/transform_clean_column_names.py,sha256=VxvWqENW63c50L96JA1V_ioe4By6gGzx_iY86njOXEM,3044
 cloe_nessy/pipeline/actions/transform_concat_columns.py,sha256=Nk8YbhxDnFZsWzW9Dj5Yl76Uq6VrcMlevQPHGms65L8,3777
 cloe_nessy/pipeline/actions/transform_decode.py,sha256=JajMwHREtxa8u_1Q3RZDBVMjncoSel-WzQFVTO0MREg,4455
-cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=E0ypz9qkHMSatNfnHekP-E6svQVL149M4PV02M03drg,5099
+cloe_nessy/pipeline/actions/transform_deduplication.py,sha256=SfTDrOL0TNSC4wITbozabC0jYvceTLnqU4urnEjYk9g,4910
 cloe_nessy/pipeline/actions/transform_distinct.py,sha256=c7aBxANyqT4aKhm0cSELDtD-bP0Se9vxlBF0K4AgQWs,1976
 cloe_nessy/pipeline/actions/transform_filter.py,sha256=Nz_ggRfKIcNzYFfFOsgq1QeatjdEis0up4I7cOWBdyo,1446
 cloe_nessy/pipeline/actions/transform_generic_sql.py,sha256=_naWfmPdYAUKjPNeHu5qJAohOL7DHCSYz_kwoeRv3OI,2741
 cloe_nessy/pipeline/actions/transform_group_aggregate.py,sha256=KUHeeP-RIDi34dpbsPEJkzea5zFJA6MuyjNpOsFud9o,4045
-cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=heRjBA-Gfu-nmNHOjTYlipEpKY8oNPAHAY40vjJk3aI,8383
+cloe_nessy/pipeline/actions/transform_hash_columns.py,sha256=H8j_Xadnm3npVNA_nu7Be7v0bJV20ELKMxSsVHHl6CY,8407
 cloe_nessy/pipeline/actions/transform_join.py,sha256=e_tvMk8YJTAWcUK_EmOgNt0s31ICZoMX_MKOTWx4lBY,3645
 cloe_nessy/pipeline/actions/transform_json_normalize.py,sha256=petF7pnNq1EKc8MqVdG0weFALAHNILSe_eAu4Z5XxIo,4833
 cloe_nessy/pipeline/actions/transform_rename_columns.py,sha256=4zJcPCONMU4C67qeuzsrX3AORRRHoq_selUI7FJyeg0,1952
 cloe_nessy/pipeline/actions/transform_replace_values.py,sha256=1OPHTrjcphfyGepcO7ozYfeqfwA18pjlyHpVKUS_AAU,2049
 cloe_nessy/pipeline/actions/transform_select_columns.py,sha256=-GhSEsb7iNnZIsYRm3BG9BX4_qUDJMbpj1DsKPY046w,4574
 cloe_nessy/pipeline/actions/transform_union.py,sha256=s81Vge0AbYPc7VkskCYfOQ_LEjqcmfNFyDkytfjcZyo,2720
-cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=j7gRuG3Fedh8JgevIFBbHKock3laJVq4l6Mx3CGU5eo,2676
-cloe_nessy/pipeline/actions/write_delta_append.py,sha256=fuL29SK9G5K14ycckU3iPexeK0XNXUfQscCwhXHxbKA,2498
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=Hir7QZZZJ9hmQZXiJ9iz6u06OCmcHFpyKFVB_I1saSM,5043
-cloe_nessy/pipeline/actions/write_file.py,sha256=H8LRst045yij-8XJ5pRB9m5d1lZpZjFa0WSVdSFesPo,2984
+cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=vZ7bZcrZY47P_EVYNshMNZ34l7Orhs8Q9--5Ud5hhLI,2906
+cloe_nessy/pipeline/actions/write_delta_append.py,sha256=2F5qnKPsY_F-2672Ce4Gub7qdna157jEqHHc429fO2A,2962
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=kQE4xLbVEUnpYImZLnpZxp88Tuf6VNSeU1W-zI8Wuvw,5805
+cloe_nessy/pipeline/actions/write_file.py,sha256=JZ8UZslxUn_ttYt5wDyvtHFq2FqYk3vOR8kvExJI8pk,3212
+cloe_nessy/pipeline/utils/__init__.py,sha256=xi02UjBMiXWD7b9gDvww4gyRyowb0eRd_6Wbu0F_cro,118
+cloe_nessy/pipeline/utils/delta_load_utils.py,sha256=KitMNruxePEkecI0h4Jint1JwJpaEog5mCOchMkgan8,1495
 cloe_nessy/session/__init__.py,sha256=t7_YjUhJYW3km_FrucaUdbIl1boQtwkyhw_8yE10qzc,74
 cloe_nessy/session/session_manager.py,sha256=VCUPhACeN5armd4D0TqDeH4Ih9nu6XvXSREFqHUwt4s,9710
 cloe_nessy/settings/__init__.py,sha256=ZbkneO3WaKOxon7qHFHnou7EnBOSnBFyKMDZblIEvzM,101
 cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_UpVM,3653
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-0.3.16.6.dist-info/METADATA,sha256=YfBuBVqeRWjBTWlj4SQKyUVrc-PX78fK_MnHhO2MQv4,3292
-cloe_nessy-0.3.16.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cloe_nessy-0.3.16.6.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
-cloe_nessy-0.3.16.6.dist-info/RECORD,,
+cloe_nessy-0.3.16.7.dist-info/METADATA,sha256=Rz35trsVqlMw7hkcDBWFr-f9EOtqII9EwZogBABy6gw,3292
+cloe_nessy-0.3.16.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cloe_nessy-0.3.16.7.dist-info/top_level.txt,sha256=Z7izn8HmQpg2wBUb-0jzaKlYKMU7Ypzuc9__9vPtW_I,11
+cloe_nessy-0.3.16.7.dist-info/RECORD,,