SilverStreamingApplication 1.7__tar.gz → 1.8__tar.gz

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.7
+Version: 1.8
 Summary: initial version
 Home-page: 
 Author: 

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/com/phida/main/Operations.py

@@ -574,5 +574,5 @@ def move_dsp_to_bronze(srcDatabaseName, srcTableName, src_dsp_path, src_table_pa
     else:
         print(f"No files were copied from {dsp_path} to {bronze_path}, so skipping refresh of {srcDatabaseName}.{srcTableName}")
 
-    # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
-    # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
+        # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
+        # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/com/phida/main/SilverMerge.py

@@ -407,28 +407,37 @@ class SilverMerge:
         lower_cols = [c.lower() for c in microBatchOutputDF.columns]
         microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
 
-        # Compatible ordering across all modes:
-        # 1) HVR-only (prod behavior): latest by hvr_integ_key
-        # 2) DSP-only: latest by src_commit_time
-        # 3) Mixed HVR + DSP: DSP rows win for same PK, then latest timestamp
+        # Compatible ordering across all modes.
+        # Do not infer mode from column presence because unionByName(allowMissingColumns=True)
+        # can materialize placeholder columns with null values.
+        # Example: HVR-only batch → after union, __operation_type column exists but is all NULL.
+        #          Checking column presence alone would incorrectly route HVR data to mixed-mode ordering.
         batch_cols = set(microBatchOutputDF.columns)
-        has_hvr_key = "hvr_integ_key" in batch_cols
-        has_dsp_op = "__operation_type" in batch_cols
+        has_hvr_key = "hvr_integ_key" in batch_cols          # True for HVR and mixed tables
+        has_dsp_op = "__operation_type" in batch_cols         # True for DSP and mixed tables (may be all-null after union)
+        has_src_commit_time = "src_commit_time" in batch_cols # Common timestamp column for both sources
 
         if has_hvr_key and not has_dsp_op:
-            # HVR-only: keep existing prod behavior
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
-        elif has_hvr_key and has_dsp_op:
-            # Mixed mode: prioritize DSP records, then newest timestamp
-            is_dsp_record = col("hvr_integ_key").isNull()
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(
-                is_dsp_record.desc(),
-                col("src_commit_time").desc(), #__timestamp is casted to src_commit_time in silverCleansing()
-                col("hvr_integ_key").desc()
-            )
+            # HVR-only table (no DSP column at all): deduplicate by integ_key descending — highest key = latest change.
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc_nulls_last())
         else:
-            # DSP-only (or no hvr_integ_key): latest timestamp wins
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc()) #__timestamp is casted to src_commit_time in silverCleansing()
+            # Mixed or DSP path: build ordering dynamically based on which recency columns actually exist.
+            order_exprs = []
+            if has_dsp_op:
+                # Flag actual DSP rows (op in A/I/L/X = 1) vs HVR rows or union-placeholder nulls (= 0).
+                # DSP rows sort first (desc), so DSP always wins over HVR for the same key in a mixed batch.
+                is_dsp_record = when(col("__operation_type").isin("A", "I", "L", "X"), lit(1)).otherwise(lit(0))
+                order_exprs.append(is_dsp_record.desc())
+            if has_src_commit_time:
+                # Within the same source type, pick the most recent record by commit timestamp.
+                order_exprs.append(col("src_commit_time").desc_nulls_last())
+            if has_hvr_key:
+                # Final tiebreaker for HVR rows: higher integ_key = later change from source.
+                order_exprs.append(col("hvr_integ_key").desc_nulls_last())
+            if not order_exprs:
+                # Defensive fallback: no recency column available, treat all rows as equal.
+                order_exprs.append(lit(1))
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(*order_exprs)
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
@@ -436,24 +445,60 @@ class SilverMerge:
         if self.enable_delete_filter == 'Y':
             microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
-        # DSP_Change: Build merge conditions only from columns that actually exist in the batch.
-        # Referencing s.__operation_type when the source is HVR-only causes DELTA_MERGE_UNRESOLVED_EXPRESSION.
+        # Build delete/update conditions only from operation columns present in the batch to avoid UNRESOLVED_EXPRESSION errors.
         delete_parts = []
+        update_insert_parts = []
+
+        # HVR: source_operation 0=delete, 3=before-image; all others are insert/update.
         if has_source_operation:
-            delete_parts.append("s.source_operation in (0,3)")
+            delete_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation in (0,3))"
+            )
+            update_insert_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation not in (0,3))"
+            )
+
+        # DSP: __operation_type X=delete; A/I/L are insert/update variants.
         if has_operation_type:
-            delete_parts.append("s.__operation_type in ('X')")
+            delete_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('X'))"
+            )
+            update_insert_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('A','I','L'))"
+            )
 
-        if not delete_parts:
+        # At least one operation column must exist; otherwise the batch is malformed.
+        if not delete_parts or not update_insert_parts:
             raise Exception("No operation column found in source batch")
 
-        delete_condition = " OR ".join([f"({p})" for p in delete_parts])
-        update_insert_condition = f"NOT ({delete_condition})"
+        # OR joins HVR and DSP branches because a row belongs to exactly one source system.
+        delete_condition = " OR ".join(delete_parts)
+        update_insert_condition = " OR ".join(update_insert_parts)
+
+        # Freshness guard: prevent a late/older microbatch from overwriting a newer target row.
+        # src_commit_time is the common recency column populated for both HVR and DSP flows.
+        # Example:
+        #   Batch 1 (arrives first):  src_commit_time = '2024-01-01 10:00' → written to Silver (t.src_commit_time = '2024-01-01 10:00')
+        #   Batch 2 (arrives late):   src_commit_time = '2024-01-01 08:00' → BLOCKED because s.src_commit_time < t.src_commit_time
+        #   Batch 3 (arrives later):  src_commit_time = '2024-01-01 12:00' → ALLOWED because s.src_commit_time >= t.src_commit_time
+        #   New key (no target row):  t.src_commit_time IS NULL             → ALLOWED (first-time insert)
+        freshness_condition = """
+        (
+            t.src_commit_time IS NULL
+            OR (s.src_commit_time IS NOT NULL AND s.src_commit_time >= t.src_commit_time)
+        )
+        """
+
+        # Apply freshness to matched delete so stale deletes cannot remove newer target rows.
+        matched_delete_condition = f"({delete_condition}) AND ({freshness_condition})"
+
+        # Apply freshness to matched update so older records do not overwrite newer silver state.
+        matched_update_condition = f"({update_insert_condition}) AND ({freshness_condition})"
 
         print(f"[SilverMerge] Executing merge for batch {batchId} into {self.tgtDatabaseName}.{self.tgtTableName}...")
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
-            .whenMatchedDelete(delete_condition) \
-            .whenMatchedUpdate(condition=update_insert_condition, set=self.columnsDict) \
+            .whenMatchedDelete(matched_delete_condition) \
+            .whenMatchedUpdate(condition=matched_update_condition, set=self.columnsDict) \
             .whenNotMatchedInsert(condition=update_insert_condition, values=self.columnsDict) \
             .execute()
         print(f"[SilverMerge] Merge for batch {batchId} complete.")

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -141,7 +141,9 @@ class WorkflowBuilder:
                     "enable_delete_filter": str(enable_delete_filter),
                     "enable_autoloader": enable_autoloader,
                     "modified_after": modified_after,
-                    "backfill_interval": backfill_interval
+                    "backfill_interval": backfill_interval,
+                    "is_dsp_kernel": is_dsp_kernel,
+                    "src_dsp_path": src_dsp_path
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -360,11 +362,19 @@ class WorkflowBuilder:
                     backfill_interval = task['backfill_interval']
                 except:
                     backfill_interval = ""
+                try:
+                    is_dsp_kernel = task['is_dsp_kernel']
+                except:
+                    is_dsp_kernel = ""
+                try:
+                    src_dsp_path = task['src_dsp_path']
+                except:
+                    src_dsp_path = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.7',
+    version='1.8',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.7 → silverstreamingapplication-1.8}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.7
+Version: 1.8
 Summary: initial version
 Home-page: 
 Author: