cloe-nessy 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py

@@ -29,6 +29,9 @@ class DeltaMergeConfig(BaseModel):
         use_partition_pruning: Flag to specify whether to use partition
             pruning to optimize the performance of the merge operation.
         partition_by: List of column names to partition by.
+        column_mapping: Mapping from target column names to source column names.
+            If a column is not in the mapping, it's assumed to have the same name
+            in both source and target.
     """
 
     dataframe_columns: list[str]
@@ -39,6 +42,7 @@ class DeltaMergeConfig(BaseModel):
     when_not_matched_insert: bool = True
     use_partition_pruning: bool = True
     partition_by: list[str] = Field(default_factory=list)
+    column_mapping: dict[str, str] = Field(default_factory=dict)
     cols_to_merge: list[str] = Field(default_factory=list, alias="_cols_to_merge")
     cols_to_update: set[str] = Field(default_factory=set, alias="_cols_to_update")
     cols_to_insert: set[str] = Field(default_factory=set, alias="_cols_to_insert")
@@ -58,11 +62,20 @@ class DeltaMergeConfig(BaseModel):
     @model_validator(mode="before")
     @classmethod
     def _validate_key_columns(cls, config: Any):
-        """Key columns must exist in the data frame."""
+        """Key columns must exist in the data frame (considering column mapping)."""
         key_columns = config.get("key_columns")
         dataframe_columns = config.get("dataframe_columns")
-        if not set(key_columns).issubset(set(dataframe_columns)):
-            raise ValueError("Key columns must exist in the DataFrame columns.")
+        column_mapping = config.get("column_mapping", {})
+
+        # For each key column (target name), find the corresponding source column
+        missing_columns = []
+        for key_col in key_columns:
+            source_col = column_mapping.get(key_col, key_col)
+            if source_col not in dataframe_columns:
+                missing_columns.append(f"{key_col} (maps to {source_col})" if key_col != source_col else key_col)
+
+        if missing_columns:
+            raise ValueError(f"Key columns must exist in the DataFrame. Missing columns: {', '.join(missing_columns)}")
         return config
 
     @model_validator(mode="before")
@@ -70,15 +83,37 @@ class DeltaMergeConfig(BaseModel):
     def _derive_merge_columns(cls, config: Any):
         """Derive update and insert columns from the DataFrame columns."""
         dataframe_columns = config.get("dataframe_columns", [])
-        config["_cols_to_merge"] = list(set(dataframe_columns))
+        column_mapping = config.get("column_mapping", {})
+
+        # Build reverse mapping: source_col -> target_col
+        reverse_mapping = {v: k for k, v in column_mapping.items()}
+
+        # Determine which target columns we're working with
+        # For each dataframe column, find its corresponding target column
+        target_columns = []
+        for df_col in dataframe_columns:
+            target_col = reverse_mapping.get(df_col, df_col)
+            target_columns.append(target_col)
+
+        config["_cols_to_merge"] = list(set(target_columns))
+
         if config.get("cols_to_exclude_from_update"):
             config["_cols_to_update"] = set(config["_cols_to_merge"]) - set(config["cols_to_exclude_from_update"])
         else:
             config["_cols_to_update"] = set(config["_cols_to_merge"])
 
         config["_cols_to_insert"] = config["_cols_to_merge"]
-        config["final_cols_to_update"] = {col: f"source.{col}" for col in config["_cols_to_update"]}
-        config["final_cols_to_insert"] = {col: f"source.{col}" for col in config["_cols_to_insert"]}
+
+        # Build final mappings using column_mapping (target -> source)
+        # For each target column, find the corresponding source column
+        config["final_cols_to_update"] = {
+            target_col: f"source.`{column_mapping.get(target_col, target_col)}`"
+            for target_col in config["_cols_to_update"]
+        }
+        config["final_cols_to_insert"] = {
+            target_col: f"source.`{column_mapping.get(target_col, target_col)}`"
+            for target_col in config["_cols_to_insert"]
+        }
         return config
 
     @model_validator(mode="after")
@@ -127,7 +162,7 @@ class DeltaMergeWriter(BaseDeltaWriter):
 
     def _build_match_conditions(self, data_frame: DataFrame, config: DeltaMergeConfig) -> str:
         """Builds match conditions for the Delta table merge."""
-        match_conditions = self._merge_match_conditions(config.key_columns)
+        match_conditions = self._merge_match_conditions_with_mapping(config.key_columns, config.column_mapping)
         if config.use_partition_pruning:
             match_conditions_list = [match_conditions] + [
                 self._partition_pruning_conditions(data_frame, config.partition_by),
@@ -169,6 +204,11 @@ class DeltaMergeWriter(BaseDeltaWriter):
         function also supports partition pruning to optimize the performance of
         the merge operation.
 
+        When source and target tables have different column names, use the
+        `column_mapping` parameter to map target column names to source column names.
+        For any columns not in the mapping, the same name is assumed for both source
+        and target.
+
         Args:
             table: The Table object representing the Delta table.
             table_identifier: The identifier of the Delta table in the format
@@ -178,7 +218,16 @@ class DeltaMergeWriter(BaseDeltaWriter):
             ignore_empty_df: A flag indicating whether to ignore an empty source
                 dataframe.
             kwargs: Passed to the
-                [`DeltaMergeConfig`][cloe_nessy.integration.writer.delta_merge_writer.DeltaMergeConfig].
+                [`DeltaMergeConfig`][cloe_nessy.integration.writer.delta_writer.delta_merge_writer.DeltaMergeConfig].
+                Common kwargs include:
+                - key_columns: List of target column names to use as merge keys.
+                - column_mapping: Dict mapping target column names to source column names.
+                - when_matched_update: Whether to update matching records.
+                - when_matched_delete: Whether to delete matching records.
+                - when_not_matched_insert: Whether to insert non-matching records.
+                - cols_to_exclude_from_update: Target columns to exclude from updates.
+                - use_partition_pruning: Whether to use partition pruning.
+                - partition_by: List of partition columns.
 
         Raises:
             ValueError: If both, table and table_identifier or storage_path are provided.
@@ -189,6 +238,21 @@ class DeltaMergeWriter(BaseDeltaWriter):
                 merge operation.
             ValueError: If partition columns are not specified when using
                 partition pruning.
+
+        Example:
+            ```python
+            # Merge with different column names
+            writer.write(
+                data_frame=source_df,
+                table=target_table,
+                key_columns=["customer_id"],
+                column_mapping={
+                    "customer_id": "cust_id",
+                    "full_name": "name",
+                    "email_address": "email"
+                }
+            )
+            ```
         """
         if self._empty_dataframe_check(data_frame, ignore_empty_df):
             return

cloe_nessy/integration/writer/delta_writer/delta_writer_base.py

@@ -150,6 +150,42 @@ class BaseDeltaWriter(BaseWriter, ABC):
         """
         return " AND ".join([f"target.`{c}` <=> source.`{c}`" for c in columns])
 
+    @staticmethod
+    def _merge_match_conditions_with_mapping(
+        key_columns: list[str], column_mapping: dict[str, str] | None = None
+    ) -> str:
+        """Merges match conditions with support for column name mapping.
+
+        This function generates SQL match conditions for merging tables where source and target
+        columns may have different names.
+
+        Args:
+            key_columns: A list of target column names to use as keys for the merge operation.
+            column_mapping: A dictionary mapping target column names to source column names.
+                If None or empty, assumes source and target columns have the same names.
+
+        Returns:
+            A string containing the match conditions, separated by " AND "
+
+        Example:
+            ```python
+            # Without mapping (same column names):
+            _merge_match_conditions_with_mapping(["id", "customer_id"])
+            # "target.`id` <=> source.`id` AND target.`customer_id` <=> source.`customer_id`"
+
+            # With mapping (different column names):
+            _merge_match_conditions_with_mapping(
+                ["id", "customer_id"],
+                {"customer_id": "cust_id"}
+            )
+            # "target.`id` <=> source.`id` AND target.`customer_id` <=> source.`cust_id`"
+            ```
+        """
+        mapping = column_mapping or {}
+        return " AND ".join(
+            [f"target.`{target_col}` <=> source.`{mapping.get(target_col, target_col)}`" for target_col in key_columns]
+        )
+
     @staticmethod
     def _partition_pruning_conditions(df: "DataFrame", partition_cols: list[str] | None) -> str:
         """Generates partition pruning conditions for an SQL query.

cloe_nessy/pipeline/actions/write_delta_merge.py

@@ -13,6 +13,7 @@ class WriteDeltaMergeAction(PipelineAction):
 
     Example:
         ```yaml
+        # Basic merge with same column names
         Write Delta Merge:
             action: WRITE_DELTA_MERGE
             options:
@@ -20,13 +21,25 @@ class WriteDeltaMergeAction(PipelineAction):
                 key_columns:
                     - id
                     - customer_id
-                cols_to_update:
-                    - name
-                    - email
-                    - updated_at
+                cols_to_exclude_from_update:
+                    - created_at
                 when_matched_update: true
                 when_not_matched_insert: true
                 use_partition_pruning: true
+
+        # Merge with different source and target column names
+        Write Delta Merge with Mapping:
+            action: WRITE_DELTA_MERGE
+            options:
+                table_identifier: my_catalog.my_schema.my_table
+                key_columns:
+                    - customer_id
+                column_mapping:
+                    customer_id: cust_id
+                    full_name: name
+                    email_address: email
+                when_matched_update: true
+                when_not_matched_insert: true
         ```
     """
 
@@ -38,11 +51,10 @@ class WriteDeltaMergeAction(PipelineAction):
         *,
         table_identifier: str | None = None,
         key_columns: list[str] | None = None,
-        cols_to_update: list[str] | None = None,
-        cols_to_insert: list[str] | None = None,
-        cols_to_exclude: list[str] | None = None,
+        cols_to_exclude_from_update: list[str] | None = None,
+        column_mapping: dict[str, str] | None = None,
         when_matched_update: bool = True,
-        when_matched_deleted: bool = False,
+        when_matched_delete: bool = False,
         when_not_matched_insert: bool = True,
         use_partition_pruning: bool = True,
         ignore_empty_df: bool = False,
@@ -57,23 +69,23 @@ class WriteDeltaMergeAction(PipelineAction):
             table_identifier: The identifier of the table. If passed, the
                 UC Adapter will be used to create a table object. Otherwise the Table
                 object will be created from the table metadata in the context.
-            key_columns: List of column names that form the
+            key_columns: List of target column names that form the
                 key for the merge operation.
+            cols_to_exclude_from_update: List of target column names to be
+                excluded from the update operation in the target Delta table.
+            column_mapping: Mapping from target column names to source column names.
+                Use this when source and target tables have different column names.
+                If a column is not in the mapping, it's assumed to have the same name
+                in both source and target.
             when_matched_update: Flag to specify whether to
-                perform an update operation whenmatching records are found in
+                perform an update operation when matching records are found in
                 the target Delta table.
-            when_matched_deleted: Flag to specify whether to
+            when_matched_delete: Flag to specify whether to
                 perform a delete operation when matching records are found in
                 the target Delta table.
             when_not_matched_insert: Flag to specify whether to perform an
                 insert operation when matching records are not found in the target
                 Delta table.
-            cols_to_update: List of column names to be
-                updated in the target Delta table.
-            cols_to_insert: List of column names to be
-                inserted into the target Delta table.
-            cols_to_exclude: List of column names to be
-                excluded from the merge operation.
             use_partition_pruning: Flag to specify whether to use partition
                 pruning to optimize the performance of the merge operation.
             ignore_empty_df: A flag indicating whether to ignore an empty source dataframe.
@@ -113,16 +125,15 @@ class WriteDeltaMergeAction(PipelineAction):
         assert key_columns is not None, "Key columns must be provided."
 
         delta_merge_writer.write(
-            table_identifier=context.table_metadata.identifier,
+            data_frame=context.data,
             table=context.table_metadata,
+            table_identifier=context.table_metadata.identifier,
             storage_path=str(context.table_metadata.storage_path),
-            data_frame=context.data,
             key_columns=key_columns,
-            cols_to_update=cols_to_update,
-            cols_to_insert=cols_to_insert,
-            cols_to_exclude=cols_to_exclude,
+            cols_to_exclude_from_update=cols_to_exclude_from_update or [],
+            column_mapping=column_mapping or {},
             when_matched_update=when_matched_update,
-            when_matched_deleted=when_matched_deleted,
+            when_matched_delete=when_matched_delete,
             when_not_matched_insert=when_not_matched_insert,
             use_partition_pruning=use_partition_pruning,
             partition_by=context.table_metadata.partition_by,

{cloe_nessy-1.0.1.dist-info → cloe_nessy-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloe-nessy
-Version: 1.0.1
+Version: 1.0.3
 Summary: Your friendly datalake monster.
 Project-URL: homepage, https://initions.com/
 Author-email: initions <ICSMC_EXT_PYPIORG@accenture.com>

{cloe_nessy-1.0.1.dist-info → cloe_nessy-1.0.3.dist-info}/RECORD

@@ -40,9 +40,9 @@ cloe_nessy/integration/writer/file_writer.py,sha256=SUDbN13ZzDhbM8DpOGFgM_Gkg70T
 cloe_nessy/integration/writer/writer.py,sha256=elFPLFrWR-qVE9qnBtzzzhyRALLQcRVuOsPS0rNmRt4,1741
 cloe_nessy/integration/writer/delta_writer/__init__.py,sha256=h2CT6Hllmk0nodlek27uqwniCzVZKMkYcPGyG9K2Z24,164
 cloe_nessy/integration/writer/delta_writer/delta_append_writer.py,sha256=nribgHmapp59v3Rw_AfJg0_BRYhP7x2IJIeE74Ia_6A,4748
-cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=Yp_q_ycasW2_wwmzty_6fZeBVcW_0o8gLrr6F1gaUjQ,10195
+cloe_nessy/integration/writer/delta_writer/delta_merge_writer.py,sha256=aMpWa8GcnW9xu5eGE_AsVyfkL5hRIeJwfCLPniM8lak,13170
 cloe_nessy/integration/writer/delta_writer/delta_table_operation_type.py,sha256=m4YFY9_WgaOcnpBviVt3Km-w3wf3NF25wPS-n0NBGcE,970
-cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=O7hw7YOa2FEzBlzjwPfxQTxm0ZrlszIjjfsHTwE_OhU,8609
+cloe_nessy/integration/writer/delta_writer/delta_writer_base.py,sha256=B7PwPHKrsJL0ZxBT-H9wWSy0gn7shqNDJ0AbrpMHyMg,10135
 cloe_nessy/integration/writer/delta_writer/exceptions.py,sha256=xPmGiYV0xQXauln5Oh34E5vbm0rVcs6xCh-SJSb2bw0,107
 cloe_nessy/logging/__init__.py,sha256=ySVCVbdyR3Dno_tl2ZfiER_7EVaDoQMHVkNyfdMZumY,65
 cloe_nessy/logging/logger_mixin.py,sha256=H8MyMEyb_kEDP0Ow5QStAFLuOkTIeUnneGaj916fKlU,7443
@@ -101,7 +101,7 @@ cloe_nessy/pipeline/actions/transform_union.py,sha256=SZtEzh567CIExUj9yMEgshE28h
 cloe_nessy/pipeline/actions/transform_with_column.py,sha256=c-E1yYkeYmovbN1maT7ImpdQlW0nYvYsHCtDvfe4wt8,3357
 cloe_nessy/pipeline/actions/write_catalog_table.py,sha256=FyC0scQU8Ul3Uigpk6IN2IJpf_4jRjAqF5yHtDVwG00,4852
 cloe_nessy/pipeline/actions/write_delta_append.py,sha256=e1g4mDhwAZdKyt4Gb7ZzHcQrJ1duSl8qOn6ONizRsoM,2934
-cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=fwinlTeZoDuTyrbln5vMu1UJ1LG8ZQrus3LoCVF__I4,5819
+cloe_nessy/pipeline/actions/write_delta_merge.py,sha256=kZL2PTIwB6Mj4UKg5f9SvU1VaakuYfFoymlcLf-L7dA,6443
 cloe_nessy/pipeline/actions/write_file.py,sha256=JZ8UZslxUn_ttYt5wDyvtHFq2FqYk3vOR8kvExJI8pk,3212
 cloe_nessy/pipeline/utils/__init__.py,sha256=xi02UjBMiXWD7b9gDvww4gyRyowb0eRd_6Wbu0F_cro,118
 cloe_nessy/pipeline/utils/delta_load_utils.py,sha256=KitMNruxePEkecI0h4Jint1JwJpaEog5mCOchMkgan8,1495
@@ -113,6 +113,6 @@ cloe_nessy/settings/settings.py,sha256=I4n129lrujriW-d8q4as2Kb4_kI932ModfZ5Ow_Up
 cloe_nessy/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 cloe_nessy/utils/column_names.py,sha256=dCNtm61mc5aLkY2oE4rlfN3VLCrpot6fOESjAZmCmhA,361
 cloe_nessy/utils/file_and_directory_handler.py,sha256=r2EVt9xG81p6ScaJCwETC5an6pMT6WseB0jMOR-JlpU,602
-cloe_nessy-1.0.1.dist-info/METADATA,sha256=qLn3XYfGsw2pW-pPtUUidtcHZiUtIwOESWY8LCenGEY,3291
-cloe_nessy-1.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-cloe_nessy-1.0.1.dist-info/RECORD,,
+cloe_nessy-1.0.3.dist-info/METADATA,sha256=fqBGuiBnOft_b6Q3yS_hxFPi5pqduBX7V7bBeXYwkvQ,3291
+cloe_nessy-1.0.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+cloe_nessy-1.0.3.dist-info/RECORD,,