SilverStreamingApplication 1.7__tar.gz → 1.9__tar.gz

{silverstreamingapplication-1.7 → silverstreamingapplication-1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.7
+Version: 1.9
 Summary: initial version
 Home-page: 
 Author: 

{silverstreamingapplication-1.7 → silverstreamingapplication-1.9}/com/phida/main/Operations.py

@@ -2,10 +2,6 @@ from pyspark.sql.utils import AnalysisException
 from com.phida.main.sparksession import spark
 from pyspark.sql.functions import col
 from pyspark.sql.types import _parse_datatype_string, StructType, StructField
-#Importing dbutils as it is used in file movement.
-from pyspark.dbutils import DBUtils
-
-dbutils = DBUtils(spark.sparkContext)
 
 
 def tableExists(dbName, tblName):
@@ -57,30 +53,14 @@ def getKeyCols(dbName, tblName):
     """
 
     try:
-        # To accomodate both the hvr and dsp data together and seperately
-        # To achive Below Logic:
-        #If Both HVR and DSP -> DSP Wins -> __timestamp
-        #IF Only HVR Or DSP -> Particluar wins
-        table = spark.read.table(f"{dbName}.{tblName}")
-        existing_cols = table.columns
-
-        # Determine the best column to order by
-        order_cols = [c for c in ["src_commit_time", "__timestamp"] if c in existing_cols]
-
-        if order_cols:
-            keyCols = (table
-                        .orderBy(*order_cols, ascending=False)
-                        .select("src_key_cols")
-                        .limit(1)
-                        .collect()[0][0]
-                    )
-        else:
-            keyCols = (table
+        keyCols = (spark.read.table(dbName + "." + tblName)
                         .select("src_key_cols")
                         .limit(1)
                         .collect()[0][0]
                     )
         return keyCols
+           
+ 
     except Exception:
         raise Exception(
             f"the column src_key_cols is not present in the given table {dbName}.{tblName}")
@@ -530,49 +510,4 @@ def buildJoinCondition(keyColsList):
     for column in keyColsList:
         condition = condition + "t.`" + column + "` <=> " + "s.`" + column + "` AND "
 
-    return condition[:-5]
-
-
-# A Function for moving files from DSP to Bronze for the given source table. 
-# Since, DSP Writes into two Folders "initial" and "delta", .parquet files from these two folders are recursively
-# moved into Bronze Table Location.
-def move_dsp_to_bronze(srcDatabaseName, srcTableName, src_dsp_path, src_table_path):
-    dsp_path = src_dsp_path
-    bronze_path = src_table_path
-    num_files = 0
-
-    print(f"Recursively copying all files from {dsp_path} to {bronze_path} (flattened, no hierarchy, source not deleted)")
-
-    def list_all_files(path):
-        files = []
-        for item in dbutils.fs.ls(path):
-            if item.isDir():
-                files.extend(list_all_files(item.path))
-            else:
-                files.append(item.path)
-        return files
-
-    try:
-        all_files = list_all_files(dsp_path)
-        for src_file in all_files:
-            file_name = src_file.split("/")[-1]
-            if ".sap." in file_name:
-                dest_file = f"{bronze_path}/{file_name}"
-                dbutils.fs.cp(src_file, dest_file)
-                #num_files += 1
-            else:
-                dest_file = f"{bronze_path}/{file_name}"
-                dbutils.fs.mv(src_file, dest_file)
-                num_files += 1
-    except Exception as e:
-        print(f"Failed to copy files: {e}")
-
-    if num_files > 0:
-        print(f"Copied {num_files} files from {dsp_path} to {bronze_path}, refreshing table {srcDatabaseName}.{srcTableName}")
-        spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
-        print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
-    else:
-        print(f"No files were copied from {dsp_path} to {bronze_path}, so skipping refresh of {srcDatabaseName}.{srcTableName}")
-
-    # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
-    # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
+    return condition[:-5]

{silverstreamingapplication-1.7 → silverstreamingapplication-1.9}/com/phida/main/SilverMerge.py

@@ -1,6 +1,6 @@
 from datetime import date, timedelta
 from com.phida.main.sparksession import spark, logger
-from com.phida.main.Operations import describe_table_to_schema, move_dsp_to_bronze, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
+from com.phida.main.Operations import describe_table_to_schema, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff, hasColumn
 from com.phida.main.utils import pathExists, convertStrToList
 from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max, lit, when, coalesce
@@ -28,11 +28,8 @@ class SilverMerge:
         cluster_table_path: String - Path to the cluster table
         enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
         use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
-        is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
-        src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
 
     methods:
-        move_dsp_to_bronze
         silverCleansing
         prepareTarget
         upsertToDelta
@@ -42,12 +39,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, is_dsp_kernel, src_dsp_path)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None, is_dsp_kernel='N', src_dsp_path=None):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None):
         """
         desc:
             Initialize the required class variables
@@ -67,8 +64,6 @@ class SilverMerge:
             cluster_table_path: String - Path to the cluster table
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
-            is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
-            src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
         """
 
         logger.info("phida_log: Initialising class variables")
@@ -90,16 +85,7 @@ class SilverMerge:
         self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
         self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() in ('Y') else 'N'
         self.modified_after = modified_after.strip() if modified_after else (date.today() - timedelta(days=15)).strftime('%Y-%m-%d')
-        self.backfill_interval = backfill_interval if backfill_interval else "1 month"
-        self.is_dsp_kernel = is_dsp_kernel.strip().upper() if is_dsp_kernel else 'N' #To determine if Kernel is loaded via Datasphere
-        self.src_dsp_path = src_dsp_path if src_dsp_path else None
-
-        #Run file movement between DSP and Bronze only if it's a DSP kernel, to ensure we don't have unnecessary file movements for non-DSP kernels
-        if self.is_dsp_kernel == 'Y':
-            src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
-            logger.info(f"phida_log: File movement between DSP and Bronze")
-            print(f"[SilverMerge] File movement between DSP and Bronze")
-            move_dsp_to_bronze(self.srcDatabaseName ,self.srcTableName, self.src_dsp_path, src_table_path)
+        self.backfill_interval = backfill_interval if backfill_interval else "1 week"
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -204,8 +190,10 @@ class SilverMerge:
 
                 silverDerivedColumns = silverDerivedColumns.drop("__timestamp")
                 logger.info("phida_log: Mapped DSP __timestamp to src_commit_time")
+                print("[SilverMerge] Mapped DSP __timestamp to src_commit_time")
             else:
                 logger.info("phida_log: HVR data; src_commit_time already present")
+                print("[SilverMerge] HVR data; src_commit_time already present")
 
             # Convert all column names to lower case to ensure
             # the target Delta table is created and maintained with lower-case column names.
@@ -404,31 +392,41 @@ class SilverMerge:
                 microBatchOutputDF = hvr_preprocessed_df.unionByName(dsp_df, allowMissingColumns=True)
 
         # Convert all column names to lower case just before writing to Silver
-        lower_cols = [c.lower() for c in microBatchOutputDF.columns]
-        microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
-
-        # Compatible ordering across all modes:
-        # 1) HVR-only (prod behavior): latest by hvr_integ_key
-        # 2) DSP-only: latest by src_commit_time
-        # 3) Mixed HVR + DSP: DSP rows win for same PK, then latest timestamp
+        #lower_cols = [c.lower() for c in microBatchOutputDF.columns]
+        #microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
+        "Commenting above column lowering as it is already done in silverCleansing() method"
+
+        # Compatible ordering across all modes.
+        # Do not infer mode from column presence because unionByName(allowMissingColumns=True)
+        # can materialize placeholder columns with null values.
+        # Example: HVR-only batch → after union, __operation_type column exists but is all NULL.
+        #          Checking column presence alone would incorrectly route HVR data to mixed-mode ordering.
         batch_cols = set(microBatchOutputDF.columns)
-        has_hvr_key = "hvr_integ_key" in batch_cols
-        has_dsp_op = "__operation_type" in batch_cols
+        has_hvr_key = "hvr_integ_key" in batch_cols          # True for HVR and mixed tables
+        has_dsp_op = "__operation_type" in batch_cols         # True for DSP and mixed tables (may be all-null after union)
+        has_src_commit_time = "src_commit_time" in batch_cols # Common timestamp column for both sources
 
         if has_hvr_key and not has_dsp_op:
-            # HVR-only: keep existing prod behavior
+            # HVR-only table (no DSP column at all): deduplicate by integ_key descending — highest key = latest change.
             windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
-        elif has_hvr_key and has_dsp_op:
-            # Mixed mode: prioritize DSP records, then newest timestamp
-            is_dsp_record = col("hvr_integ_key").isNull()
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(
-                is_dsp_record.desc(),
-                col("src_commit_time").desc(), #__timestamp is casted to src_commit_time in silverCleansing()
-                col("hvr_integ_key").desc()
-            )
         else:
-            # DSP-only (or no hvr_integ_key): latest timestamp wins
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc()) #__timestamp is casted to src_commit_time in silverCleansing()
+            # Mixed or DSP path: build ordering dynamically based on which recency columns actually exist.
+            order_exprs = []
+            if has_dsp_op:
+                # Flag actual DSP rows (op in A/I/L/X = 1) vs HVR rows or union-placeholder nulls (= 0).
+                # DSP rows sort first (desc), so DSP always wins over HVR for the same key in a mixed batch.
+                is_dsp_record = when(col("__operation_type").isin("A", "I", "L", "X"), lit(1)).otherwise(lit(0))
+                order_exprs.append(is_dsp_record.desc())
+            if has_src_commit_time:
+                # Within the same source type, pick the most recent record by commit timestamp.
+                order_exprs.append(col("src_commit_time").desc_nulls_last())
+            if has_hvr_key:
+                # Final tiebreaker for HVR rows: higher integ_key = later change from source.
+                order_exprs.append(col("hvr_integ_key").desc_nulls_last())
+            if not order_exprs:
+                # Defensive fallback: no recency column available, treat all rows as equal.
+                order_exprs.append(lit(1))
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(*order_exprs)
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
@@ -436,24 +434,60 @@ class SilverMerge:
         if self.enable_delete_filter == 'Y':
             microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
-        # DSP_Change: Build merge conditions only from columns that actually exist in the batch.
-        # Referencing s.__operation_type when the source is HVR-only causes DELTA_MERGE_UNRESOLVED_EXPRESSION.
+        # Build delete/update conditions only from operation columns present in the batch to avoid UNRESOLVED_EXPRESSION errors.
         delete_parts = []
+        update_insert_parts = []
+
+        # HVR: source_operation 0=delete, 3=before-image; all others are insert/update.
         if has_source_operation:
-            delete_parts.append("s.source_operation in (0,3)")
+            delete_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation in (0,3))"
+            )
+            update_insert_parts.append(
+                "(s.source_operation IS NOT NULL AND s.source_operation not in (0,3))"
+            )
+
+        # DSP: __operation_type X=delete; A/I/L are insert/update variants.
         if has_operation_type:
-            delete_parts.append("s.__operation_type in ('X')")
+            delete_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('X'))"
+            )
+            update_insert_parts.append(
+                "(s.__operation_type IS NOT NULL AND s.__operation_type in ('A','I','L'))"
+            )
 
-        if not delete_parts:
+        # At least one operation column must exist; otherwise the batch is malformed.
+        if not delete_parts or not update_insert_parts:
             raise Exception("No operation column found in source batch")
 
-        delete_condition = " OR ".join([f"({p})" for p in delete_parts])
-        update_insert_condition = f"NOT ({delete_condition})"
+        # OR joins HVR and DSP branches because a row belongs to exactly one source system.
+        delete_condition = " OR ".join(delete_parts)
+        update_insert_condition = " OR ".join(update_insert_parts)
+
+        # Freshness guard: prevent a late/older microbatch from overwriting a newer target row.
+        # src_commit_time is the common recency column populated for both HVR and DSP flows.
+        # Example:
+        #   Batch 1 (arrives first):  src_commit_time = '2024-01-01 10:00' → written to Silver (t.src_commit_time = '2024-01-01 10:00')
+        #   Batch 2 (arrives late):   src_commit_time = '2024-01-01 08:00' → BLOCKED because s.src_commit_time < t.src_commit_time
+        #   Batch 3 (arrives later):  src_commit_time = '2024-01-01 12:00' → ALLOWED because s.src_commit_time >= t.src_commit_time
+        #   New key (no target row):  t.src_commit_time IS NULL             → ALLOWED (first-time insert)
+        freshness_condition = """
+        (
+            t.src_commit_time IS NULL
+            OR (s.src_commit_time IS NOT NULL AND s.src_commit_time >= t.src_commit_time)
+        )
+        """
+
+        # Apply freshness to matched delete so stale deletes cannot remove newer target rows.
+        matched_delete_condition = f"({delete_condition}) AND ({freshness_condition})"
+
+        # Apply freshness to matched update so older records do not overwrite newer silver state.
+        matched_update_condition = f"({update_insert_condition}) AND ({freshness_condition})"
 
         print(f"[SilverMerge] Executing merge for batch {batchId} into {self.tgtDatabaseName}.{self.tgtTableName}...")
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
-            .whenMatchedDelete(delete_condition) \
-            .whenMatchedUpdate(condition=update_insert_condition, set=self.columnsDict) \
+            .whenMatchedDelete(matched_delete_condition) \
+            .whenMatchedUpdate(condition=matched_update_condition, set=self.columnsDict) \
             .whenNotMatchedInsert(condition=update_insert_condition, values=self.columnsDict) \
             .execute()
         print(f"[SilverMerge] Merge for batch {batchId} complete.")

{silverstreamingapplication-1.7 → silverstreamingapplication-1.9}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.7',
+    version='1.9',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.7 → silverstreamingapplication-1.9}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.7
+Version: 1.9
 Summary: initial version
 Home-page: 
 Author: