SilverStreamingApplication 1.6__tar.gz → 1.8__tar.gz

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.6
+Version: 1.8
 Summary: initial version
 Home-page: 
 Author: 

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/com/phida/main/Operations.py

@@ -2,6 +2,10 @@ from pyspark.sql.utils import AnalysisException
 from com.phida.main.sparksession import spark
 from pyspark.sql.functions import col
 from pyspark.sql.types import _parse_datatype_string, StructType, StructField
+#Importing dbutils as it is used in file movement.
+from pyspark.dbutils import DBUtils
+
+dbutils = DBUtils(spark.sparkContext)
 
 
 def tableExists(dbName, tblName):
@@ -49,17 +53,34 @@ def getKeyCols(dbName, tblName):
         getKeyCols("<database name>", "vbuk")
 
     tip:
-        Make sure the table is a bronze table and has the columns "src_key_cols" and "src_commit_time"
+        Make sure the table is a bronze table and has the columns "src_key_cols", "src_commit_time", and "__timestamp"
     """
+
     try:
-        
-        keyCols = (spark.read.table(dbName + "." + tblName)
-                        .select("src_key_cols", "src_commit_time")
+        # To accomodate both the hvr and dsp data together and seperately
+        # To achive Below Logic:
+        #If Both HVR and DSP -> DSP Wins -> __timestamp
+        #IF Only HVR Or DSP -> Particluar wins
+        table = spark.read.table(f"{dbName}.{tblName}")
+        existing_cols = table.columns
+
+        # Determine the best column to order by
+        order_cols = [c for c in ["src_commit_time", "__timestamp"] if c in existing_cols]
+
+        if order_cols:
+            keyCols = (table
+                        .orderBy(*order_cols, ascending=False)
+                        .select("src_key_cols")
+                        .limit(1)
+                        .collect()[0][0]
+                    )
+        else:
+            keyCols = (table
+                        .select("src_key_cols")
                         .limit(1)
                         .collect()[0][0]
-                   )
+                    )
         return keyCols
-
     except Exception:
         raise Exception(
             f"the column src_key_cols is not present in the given table {dbName}.{tblName}")
@@ -270,6 +291,13 @@ def addDerivedColumns(df, colExprList):
         except Exception:
             raise Exception("The given column expression is invalid")
 
+        # TO Handel DataTypes issues (DecFloat issues in DSP)
+        # Value: cast(CFB_ADTDAYS as decimal(4,0)) as cfb_adtdays§ cast(SICHZ_TRM as decimal(6,3))
+        # If the alias matches an existing column name (case-insensitive), drop the
+        # original first so the derived expression replaces it instead of duplicating it.
+        alias = colExpr.split(" as ")[-1].strip().lower()
+        columnList = [c for c in columnList if c.strip("`").lower() != alias]
+
         columnList.append(colExpr)
 
     dfOut = df.selectExpr(columnList)
@@ -503,3 +531,48 @@ def buildJoinCondition(keyColsList):
         condition = condition + "t.`" + column + "` <=> " + "s.`" + column + "` AND "
 
     return condition[:-5]
+
+
+# A Function for moving files from DSP to Bronze for the given source table. 
+# Since, DSP Writes into two Folders "initial" and "delta", .parquet files from these two folders are recursively
+# moved into Bronze Table Location.
+def move_dsp_to_bronze(srcDatabaseName, srcTableName, src_dsp_path, src_table_path):
+    dsp_path = src_dsp_path
+    bronze_path = src_table_path
+    num_files = 0
+
+    print(f"Recursively copying all files from {dsp_path} to {bronze_path} (flattened, no hierarchy, source not deleted)")
+
+    def list_all_files(path):
+        files = []
+        for item in dbutils.fs.ls(path):
+            if item.isDir():
+                files.extend(list_all_files(item.path))
+            else:
+                files.append(item.path)
+        return files
+
+    try:
+        all_files = list_all_files(dsp_path)
+        for src_file in all_files:
+            file_name = src_file.split("/")[-1]
+            if ".sap." in file_name:
+                dest_file = f"{bronze_path}/{file_name}"
+                dbutils.fs.cp(src_file, dest_file)
+                #num_files += 1
+            else:
+                dest_file = f"{bronze_path}/{file_name}"
+                dbutils.fs.mv(src_file, dest_file)
+                num_files += 1
+    except Exception as e:
+        print(f"Failed to copy files: {e}")
+
+    if num_files > 0:
+        print(f"Copied {num_files} files from {dsp_path} to {bronze_path}, refreshing table {srcDatabaseName}.{srcTableName}")
+        spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
+        print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
+    else:
+        print(f"No files were copied from {dsp_path} to {bronze_path}, so skipping refresh of {srcDatabaseName}.{srcTableName}")
+
+        # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
+        # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/com/phida/main/SilverMerge.py

@@ -1,9 +1,9 @@
 from datetime import date, timedelta
 from com.phida.main.sparksession import spark, logger
-from com.phida.main.Operations import describe_table_to_schema, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
-    dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
+from com.phida.main.Operations import describe_table_to_schema, move_dsp_to_bronze, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
+    dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff, hasColumn
 from com.phida.main.utils import pathExists, convertStrToList
-from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max
+from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max, lit, when, coalesce
 from pyspark.sql.window import Window
 from delta.tables import DeltaTable
 from pyspark.sql import DataFrame
@@ -28,8 +28,11 @@ class SilverMerge:
         cluster_table_path: String - Path to the cluster table
         enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
         use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
+        is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
+        src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
 
     methods:
+        move_dsp_to_bronze
         silverCleansing
         prepareTarget
         upsertToDelta
@@ -39,12 +42,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, is_dsp_kernel, src_dsp_path)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None, is_dsp_kernel='N', src_dsp_path=None):
         """
         desc:
             Initialize the required class variables
@@ -64,9 +67,13 @@ class SilverMerge:
             cluster_table_path: String - Path to the cluster table
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
+            is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
+            src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
         """
 
         logger.info("phida_log: Initialising class variables")
+        logger.info("Reading Bronze Table...")
+        print(f"[SilverMerge] Initialising pipeline for {srcDatabaseName}.{srcTableName} -> {tgtDatabaseName}.{tgtTableName}")
 
         self.srcDatabaseName = srcDatabaseName
         self.srcTableName = srcTableName
@@ -84,35 +91,62 @@ class SilverMerge:
         self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() in ('Y') else 'N'
         self.modified_after = modified_after.strip() if modified_after else (date.today() - timedelta(days=15)).strftime('%Y-%m-%d')
         self.backfill_interval = backfill_interval if backfill_interval else "1 month"
+        self.is_dsp_kernel = is_dsp_kernel.strip().upper() if is_dsp_kernel else 'N' #To determine if Kernel is loaded via Datasphere
+        self.src_dsp_path = src_dsp_path if src_dsp_path else None
+
+        #Run file movement between DSP and Bronze only if it's a DSP kernel, to ensure we don't have unnecessary file movements for non-DSP kernels
+        if self.is_dsp_kernel == 'Y':
+            src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
+            logger.info(f"phida_log: File movement between DSP and Bronze")
+            print(f"[SilverMerge] File movement between DSP and Bronze")
+            move_dsp_to_bronze(self.srcDatabaseName ,self.srcTableName, self.src_dsp_path, src_table_path)
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
         if tableExists(self.srcDatabaseName, self.srcTableName):
+            logger.info(f"Bronze table {self.srcDatabaseName}.{self.srcTableName} exists.")
+            print(f"[SilverMerge] Bronze table {self.srcDatabaseName}.{self.srcTableName} found.")
 
             logger.info(f"phida_log: source table exists")
 
             logger.info(f"phida_log: initialising derived class variables")
 
             if self.use_auto_loader == 'N':
+                logger.info("Extracting Bronze Structure using Spark readStream.table...")
+                print("[SilverMerge] Reading Bronze table using Spark readStream...")
                 self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+                logger.info("Bronze Structure extracted.")
+                print("[SilverMerge] Bronze structure extracted.")
             elif self.use_auto_loader == 'Y':
+                logger.info("Extracting Bronze Structure using Auto Loader...")
+                print("[SilverMerge] Reading Bronze table using Auto Loader...")
                 desc = spark.sql(f"DESCRIBE TABLE {self.srcDatabaseName}.{self.srcTableName}").collect()
                 schema = describe_table_to_schema(desc)
                 src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
                 self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").option("modifiedAfter", f"{self.modified_after} 00:00:00.000000 UTC+0").option("cloudFiles.backfillInterval", self.backfill_interval).schema(schema).load(src_table_path).drop("_rescued_data")
+                logger.info("Bronze Structure extracted.")
+                print("[SilverMerge] Bronze structure extracted via Auto Loader.")
 
+            logger.info("Determining key columns for merge...")
+            print("[SilverMerge] Determining key columns for merge...")
             self.keyCols = getKeyCols(self.srcDatabaseName, self.srcTableName)
 
             self.keyColsList = convertStrToList(self.keyCols, ",")
-
+            logger.info(f"Key columns for merge: {self.keyColsList}")
+            print(f"[SilverMerge] Key columns: {self.keyColsList}")
+            
             self.joinCondition = buildJoinCondition(self.keyColsList)
+            logger.info(f"Join condition for merge: {self.joinCondition}")
 
             self.condition = f"{self.pruneColumn} <> '' AND {self.joinCondition}" if self.pruneColumn.strip() \
                              else self.joinCondition
+            logger.info(f"Final merge condition: {self.condition}")
 
             self.dropColumnList = convertStrToList(self.dropColumnStr, ",")
 
             self.columnsDict = buildColumnsDict(self.srcDF, self.dropColumnList)
+            logger.info("Initialization of SilverMerge complete.")
+            print("[SilverMerge] Initialization complete.")
 
     def silverCleansing(self):
         """
@@ -132,21 +166,56 @@ class SilverMerge:
             N/A
         """
         logger.info(f"phida_log: applying cleansing rules on source dataframe ")
+        logger.info("Starting cleansing transformations on Bronze DataFrame...")
+        print("[SilverMerge] Starting Silver cleansing transformations...")
+
 
         if tableExists(self.srcDatabaseName, self.srcTableName):
 
             silverRawDF = self.srcDF
+            logger.info("Bronze DataFrame loaded for cleansing.")
 
             if self.derivedColExpr:
+                logger.info("Adding derived columns to DataFrame...")
                 derivedColExprList = convertStrToList(self.derivedColExpr, "§")
-
                 silverDerivedColumns = addDerivedColumns(silverRawDF, derivedColExprList)
-
+                logger.info("Derived columns added.")
             else:
                 silverDerivedColumns = silverRawDF
+                logger.info("No derived columns specified.")
+
+            # Map DSP __timestamp (timestamp_ntz) to src_commit_time for unified timestamp handling.
+            # And drop __timestamp from the DF
+            # Handle both mixed HVR+DSP and DSP-only inputs safely.
+            has_src_commit_time = "src_commit_time" in silverDerivedColumns.columns
+            has_dsp_timestamp = "__timestamp" in silverDerivedColumns.columns
+
+            if has_dsp_timestamp:
+                if has_src_commit_time:
+                    silverDerivedColumns = silverDerivedColumns.withColumn(
+                        "src_commit_time",
+                        coalesce(col("src_commit_time"), col("__timestamp").cast("timestamp"))
+                    )
+                else:
+                    silverDerivedColumns = silverDerivedColumns.withColumn(
+                        "src_commit_time",
+                        col("__timestamp").cast("timestamp")
+                    )
+
+                silverDerivedColumns = silverDerivedColumns.drop("__timestamp")
+                logger.info("phida_log: Mapped DSP __timestamp to src_commit_time")
+            else:
+                logger.info("phida_log: HVR data; src_commit_time already present")
 
-            self.columnsDict = buildColumnsDict(silverDerivedColumns, self.dropColumnList)
+            # Convert all column names to lower case to ensure
+            # the target Delta table is created and maintained with lower-case column names.
+            silverDerivedColumns = silverDerivedColumns.toDF(*[c.lower() for c in silverDerivedColumns.columns])
+            logger.info("Column names converted to lower case.")
+            print("[SilverMerge] Column names normalised to lower case.")
 
+            self.columnsDict = buildColumnsDict(silverDerivedColumns, self.dropColumnList)
+            logger.info("Cleansing transformations complete.")
+            print("[SilverMerge] Cleansing transformations complete.")
             return silverDerivedColumns
 
     def prepareTarget(self, inDF):
@@ -171,6 +240,7 @@ class SilverMerge:
             N/A
         """
         logger.info(f"phida_log: preparing the target delta table ")
+        print(f"[SilverMerge] Preparing target table {self.tgtDatabaseName}.{self.tgtTableName}...")
 
         targetTableExists = tableExists(self.tgtDatabaseName, self.tgtTableName)
 
@@ -184,6 +254,7 @@ class SilverMerge:
 
             logger.info(f"phida_log: This seems to be the first run")
             logger.info(f"phida_log: creating the target table {self.tgtDatabaseName}.{self.tgtTableName}")
+            print(f"[SilverMerge] First run detected. Creating target table {self.tgtDatabaseName}.{self.tgtTableName}...")
 
             createDeltaTable(inDF,
                              self.tgtTablePath,
@@ -212,6 +283,7 @@ class SilverMerge:
             if addColumns:
                 logger.info(f"phida_log: There seems to be a schema change in silver")
                 logger.info(f"phida_log: Altering the target table {self.tgtDatabaseName}.{self.tgtTableName}")
+                print(f"[SilverMerge] Schema change detected. Altering target table with new columns: {addColumns}")
 
                 alterDeltaTable(self.tgtDatabaseName, self.tgtTableName, addColumns)
 
@@ -219,37 +291,38 @@ class SilverMerge:
 
             else:
                 logger.info(f"phida_log: There is no change in schema in silver")
+                print("[SilverMerge] No schema changes detected.")
 
     def filter_deleted_records(self, microBatchOutputDF):
-           """
-            Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
-            within the last 4 hours based on hvr_integ_key.
-           """
+        """
+        Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
+        within the last 4 hours based on hvr_integ_key.
+        """
            
-           if microBatchOutputDF.isEmpty():
-               return microBatchOutputDF
+        if microBatchOutputDF.isEmpty():
+            return microBatchOutputDF
 
-           filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
+        filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
            
-           if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
-               return microBatchOutputDF
+        if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
+            return microBatchOutputDF
         
-           delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
+        delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
 
-           if delete_records.isEmpty():
-               return microBatchOutputDF
+        if delete_records.isEmpty():
+            return microBatchOutputDF
 
-           delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
+        delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
 
-           joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
+        joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
 
-           filtered_df = (joined_df.filter(
+        filtered_df = (joined_df.filter(
                               (col("delete_hvr_integ_key").isNull()) |
                               (col("hvr_integ_key") >= col("delete_hvr_integ_key"))
                           )
                           .drop("delete_hvr_integ_key"))
 
-           return filtered_df 
+        return filtered_df 
                 
     def preprocess_dataframe(self, df: DataFrame, parent_keys: list) -> DataFrame:
         
@@ -275,8 +348,6 @@ class SilverMerge:
         unpack_keys = set(row["key_columns_for_unpack_table"].split(',')) if row["key_columns_for_unpack_table"] else set()
         parent_keys = list(pack_keys & unpack_keys)
         return parent_keys
-    
-        
 
     def upsertToDelta(self, microBatchOutputDF, batchId):
         """
@@ -296,11 +367,28 @@ class SilverMerge:
         tip:
             N/A
         """
-        op_8_df = microBatchOutputDF.filter("source_operation = 8")
-        microBatchOutputDF = microBatchOutputDF.filter("source_operation in (0,1,2,3)")
-        microBatchOutputDF_8 = microBatchOutputDF.filter("source_operation in (0,1,2,3,8)")
+
+        # DSP_Change: Support both HVR (source_operation) and DSP (__operation_type) operation types
+        # HVR: source_operation in (0,1,2,3) where 0=delete, 1=insert, 2=update, 3=before-image
+        # DSP: __operation_type in ('A','I','L','X') where A=Append, I=Delta-Insert, L=Initial Load, X=delete
+        cols = set(microBatchOutputDF.columns)
+        has_source_operation = "source_operation" in cols
+        has_operation_type = "__operation_type" in cols
+
+        empty_df = microBatchOutputDF.limit(0)
+
+        # Build filtered views once
+        hvr_df_8 = (microBatchOutputDF.filter("source_operation in (0,1,2,3,8)") if has_source_operation else empty_df)
+        op_8_df = (hvr_df_8.filter("source_operation = 8") if has_source_operation else empty_df)
+        hvr_df = (hvr_df_8.filter("source_operation in (0,1,2,3)") if has_source_operation else empty_df)
+        dsp_df = (microBatchOutputDF.filter("__operation_type in ('A','I','L','X')") if has_operation_type else empty_df)
+
+        # Keep op-8/preprocess flow strictly HVR-only. DSP records should not go through op-8 path.
+        microBatchOutputDF_8 = hvr_df_8
+        microBatchOutputDF = hvr_df.unionByName(dsp_df, allowMissingColumns=True)
+
         tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
-        
+
         if self.cluster_table_path:
             if not op_8_df.isEmpty():
                 table_details = DeltaTable.forName(spark, self.cluster_table_path)
@@ -311,9 +399,45 @@ class SilverMerge:
                     tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
                     .whenMatchedDelete() \
                     .execute()
-                microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
-
-        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
+                # Apply op-8 preprocess only to HVR rows, then add DSP rows back.
+                hvr_preprocessed_df = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
+                microBatchOutputDF = hvr_preprocessed_df.unionByName(dsp_df, allowMissingColumns=True)
+
+        # Convert all column names to lower case just before writing to Silver
+        lower_cols = [c.lower() for c in microBatchOutputDF.columns]
+        microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
+
+        # Compatible ordering across all modes.
+        # Do not infer mode from column presence because unionByName(allowMissingColumns=True)
+        # can materialize placeholder columns with null values.
+        # Example: HVR-only batch → after union, __operation_type column exists but is all NULL.
+        #          Checking column presence alone would incorrectly route HVR data to mixed-mode ordering.
+        batch_cols = set(microBatchOutputDF.columns)
+        has_hvr_key = "hvr_integ_key" in batch_cols          # True for HVR and mixed tables
+        has_dsp_op = "__operation_type" in batch_cols         # True for DSP and mixed tables (may be all-null after union)
+        has_src_commit_time = "src_commit_time" in batch_cols # Common timestamp column for both sources
+
+        if has_hvr_key and not has_dsp_op:
+            # HVR-only table (no DSP column at all): deduplicate by integ_key descending — highest key = latest change.
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc_nulls_last())
+        else:
+            # Mixed or DSP path: build ordering dynamically based on which recency columns actually exist.
+            order_exprs = []
+            if has_dsp_op:
+                # Flag actual DSP rows (op in A/I/L/X = 1) vs HVR rows or union-placeholder nulls (= 0).
+                # DSP rows sort first (desc), so DSP always wins over HVR for the same key in a mixed batch.
+                is_dsp_record = when(col("__operation_type").isin("A", "I", "L", "X"), lit(1)).otherwise(lit(0))
+                order_exprs.append(is_dsp_record.desc())
+            if has_src_commit_time:
+                # Within the same source type, pick the most recent record by commit timestamp.
+                order_exprs.append(col("src_commit_time").desc_nulls_last())
+            if has_hvr_key:
+                # Final tiebreaker for HVR rows: higher integ_key = later change from source.
+                order_exprs.append(col("hvr_integ_key").desc_nulls_last())
+            if not order_exprs:
+                # Defensive fallback: no recency column available, treat all rows as equal.
+                order_exprs.append(lit(1))
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(*order_exprs)
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
@@ -321,11 +445,63 @@ class SilverMerge:
         if self.enable_delete_filter == 'Y':
             microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
+        # Build delete/update conditions only from operation columns present in the batch to avoid UNRESOLVED_EXPRESSION errors.
+        delete_parts = []
+        update_insert_parts = []
+
+        # HVR: source_operation 0=delete, 3=before-image; all others are insert/update.
+        if has_source_operation:
+            delete_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation in (0,3))"
+            )
+            update_insert_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation not in (0,3))"
+            )
+
+        # DSP: __operation_type X=delete; A/I/L are insert/update variants.
+        if has_operation_type:
+            delete_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('X'))"
+            )
+            update_insert_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('A','I','L'))"
+            )
+
+        # At least one operation column must exist; otherwise the batch is malformed.
+        if not delete_parts or not update_insert_parts:
+            raise Exception("No operation column found in source batch")
+
+        # OR joins HVR and DSP branches because a row belongs to exactly one source system.
+        delete_condition = " OR ".join(delete_parts)
+        update_insert_condition = " OR ".join(update_insert_parts)
+
+        # Freshness guard: prevent a late/older microbatch from overwriting a newer target row.
+        # src_commit_time is the common recency column populated for both HVR and DSP flows.
+        # Example:
+        #   Batch 1 (arrives first):  src_commit_time = '2024-01-01 10:00' → written to Silver (t.src_commit_time = '2024-01-01 10:00')
+        #   Batch 2 (arrives late):   src_commit_time = '2024-01-01 08:00' → BLOCKED because s.src_commit_time < t.src_commit_time
+        #   Batch 3 (arrives later):  src_commit_time = '2024-01-01 12:00' → ALLOWED because s.src_commit_time >= t.src_commit_time
+        #   New key (no target row):  t.src_commit_time IS NULL             → ALLOWED (first-time insert)
+        freshness_condition = """
+        (
+            t.src_commit_time IS NULL
+            OR (s.src_commit_time IS NOT NULL AND s.src_commit_time >= t.src_commit_time)
+        )
+        """
+
+        # Apply freshness to matched delete so stale deletes cannot remove newer target rows.
+        matched_delete_condition = f"({delete_condition}) AND ({freshness_condition})"
+
+        # Apply freshness to matched update so older records do not overwrite newer silver state.
+        matched_update_condition = f"({update_insert_condition}) AND ({freshness_condition})"
+
+        print(f"[SilverMerge] Executing merge for batch {batchId} into {self.tgtDatabaseName}.{self.tgtTableName}...")
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
-            .whenMatchedDelete("s.source_operation in (0,3)") \
-            .whenMatchedUpdate(condition="s.source_operation not in (0,3)", set=self.columnsDict) \
-            .whenNotMatchedInsert(condition="s.source_operation not in (0,3)", values=self.columnsDict) \
+            .whenMatchedDelete(matched_delete_condition) \
+            .whenMatchedUpdate(condition=matched_update_condition, set=self.columnsDict) \
+            .whenNotMatchedInsert(condition=update_insert_condition, values=self.columnsDict) \
             .execute()
+        print(f"[SilverMerge] Merge for batch {batchId} complete.")
  
     def streamIntoDeltaTarget(self):
         """
@@ -350,6 +526,7 @@ class SilverMerge:
         self.prepareTarget(silverCleansedDF)
 
         logger.info(f"phida_log: performing streaming merge on target {self.tgtDatabaseName}.{self.tgtTableName}")
+        print(f"[SilverMerge] Starting streaming merge: {self.srcDatabaseName}.{self.srcTableName} -> {self.tgtDatabaseName}.{self.tgtTableName}")
 
         if self.triggerOnce == "Y":
             outputDF = (silverCleansedDF.writeStream
@@ -373,4 +550,4 @@ class SilverMerge:
                         .start(self.tgtTablePath)
                         )
 
-        return outputDF
+        return outputDF

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -141,7 +141,9 @@ class WorkflowBuilder:
                     "enable_delete_filter": str(enable_delete_filter),
                     "enable_autoloader": enable_autoloader,
                     "modified_after": modified_after,
-                    "backfill_interval": backfill_interval
+                    "backfill_interval": backfill_interval,
+                    "is_dsp_kernel": is_dsp_kernel,
+                    "src_dsp_path": src_dsp_path
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -360,11 +362,19 @@ class WorkflowBuilder:
                     backfill_interval = task['backfill_interval']
                 except:
                     backfill_interval = ""
+                try:
+                    is_dsp_kernel = task['is_dsp_kernel']
+                except:
+                    is_dsp_kernel = ""
+                try:
+                    src_dsp_path = task['src_dsp_path']
+                except:
+                    src_dsp_path = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.6',
+    version='1.8',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.6 → silverstreamingapplication-1.8}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.6
+Version: 1.8
 Summary: initial version
 Home-page: 
 Author: