SilverStreamingApplication 1.5__tar.gz → 1.7__tar.gz

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.5
+Version: 1.7
 Summary: initial version
 Home-page: 
 Author: 

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/com/phida/main/Operations.py

@@ -1,6 +1,11 @@
 from pyspark.sql.utils import AnalysisException
 from com.phida.main.sparksession import spark
 from pyspark.sql.functions import col
+from pyspark.sql.types import _parse_datatype_string, StructType, StructField
+#Importing dbutils as it is used in file movement.
+from pyspark.dbutils import DBUtils
+
+dbutils = DBUtils(spark.sparkContext)
 
 
 def tableExists(dbName, tblName):
@@ -48,17 +53,34 @@ def getKeyCols(dbName, tblName):
         getKeyCols("<database name>", "vbuk")
 
     tip:
-        Make sure the table is a bronze table and has the columns "src_key_cols" and "src_commit_time"
+        Make sure the table is a bronze table and has the columns "src_key_cols", "src_commit_time", and "__timestamp"
     """
+
     try:
-        
-        keyCols = (spark.read.table(dbName + "." + tblName)
-                        .select("src_key_cols", "src_commit_time")
+        # To accomodate both the hvr and dsp data together and seperately
+        # To achive Below Logic:
+        #If Both HVR and DSP -> DSP Wins -> __timestamp
+        #IF Only HVR Or DSP -> Particluar wins
+        table = spark.read.table(f"{dbName}.{tblName}")
+        existing_cols = table.columns
+
+        # Determine the best column to order by
+        order_cols = [c for c in ["src_commit_time", "__timestamp"] if c in existing_cols]
+
+        if order_cols:
+            keyCols = (table
+                        .orderBy(*order_cols, ascending=False)
+                        .select("src_key_cols")
+                        .limit(1)
+                        .collect()[0][0]
+                    )
+        else:
+            keyCols = (table
+                        .select("src_key_cols")
                         .limit(1)
                         .collect()[0][0]
-                   )
+                    )
         return keyCols
-
     except Exception:
         raise Exception(
             f"the column src_key_cols is not present in the given table {dbName}.{tblName}")
@@ -92,6 +114,89 @@ def hiveDDL(df):
 
     return tblDDL
 
+def describe_table_to_schema(desc_rows):
+    """
+    Convert table description rows into a Spark StructType schema.
+    This function parses the output from a DESCRIBE TABLE command and organizes
+    the columns into a StructType schema, separating regular data fields from
+    partition fields.
+    Args:
+        desc_rows: An iterable of row objects containing table metadata.
+                  Each row should have 'col_name' and 'data_type' attributes.
+    Returns:
+        StructType: A Spark StructType object representing the table schema,
+                   with data fields listed first, followed by partition fields.
+    Example:
+        >>> desc_rows = [
+        ...     Row(col_name='id', data_type='bigint'),
+        ...     Row(col_name='name', data_type='string'),
+        ...     Row(col_name='# Partition Information', data_type=''),
+        ...     Row(col_name='date', data_type='date')
+        ... ]
+        >>> schema = describe_table_to_schema(desc_rows)
+    """
+    data_fields = []
+    partition_fields = []
+
+    in_partition_section = False
+
+    for r in desc_rows:
+        col = r.col_name
+        dtype = r.data_type
+
+        # Enter partition section
+        if col.startswith("# Partition Information"):
+            in_partition_section = True
+            continue
+
+        # Skip header rows
+        if col.startswith("#"):
+            continue
+
+        # Skip empty rows (defensive)
+        if not col or not dtype:
+            continue
+
+        field = StructField(
+            col,
+            _parse_datatype(dtype),
+            True
+        )
+
+        if in_partition_section:
+            partition_fields.append(field)
+        else:
+            data_fields.append(field)
+
+    return StructType(data_fields + partition_fields)
+
+def _parse_datatype(dt):
+    """
+    Parse a data type string and return the corresponding Spark data type.
+    
+    This function serves as a wrapper that leverages Spark's internal parsing
+    mechanism to correctly handle complex data types including nested structures,
+    arrays, maps, and other composite types.
+    
+    Args:
+        dt (str): A string representation of a Spark data type 
+                 (e.g., 'IntegerType', 'StructType', 'ArrayType(StringType)')
+    
+    Returns:
+        pyspark.sql.types.DataType: The parsed Spark data type object corresponding
+                                    to the input string.
+    
+    Raises:
+        ParseException: If the data type string is invalid or cannot be parsed.
+    
+    Examples:
+        >>> _parse_datatype('IntegerType')
+        IntegerType
+        >>> _parse_datatype('ArrayType(StringType)')
+        ArrayType(StringType, containsNull=true)
+    """
+    # Let Spark parse complex types correctly
+    return _parse_datatype_string(dt)
 
 def hasColumn(df, columnName):
     """
@@ -186,6 +291,13 @@ def addDerivedColumns(df, colExprList):
         except Exception:
             raise Exception("The given column expression is invalid")
 
+        # TO Handel DataTypes issues (DecFloat issues in DSP)
+        # Value: cast(CFB_ADTDAYS as decimal(4,0)) as cfb_adtdays§ cast(SICHZ_TRM as decimal(6,3))
+        # If the alias matches an existing column name (case-insensitive), drop the
+        # original first so the derived expression replaces it instead of duplicating it.
+        alias = colExpr.split(" as ")[-1].strip().lower()
+        columnList = [c for c in columnList if c.strip("`").lower() != alias]
+
         columnList.append(colExpr)
 
     dfOut = df.selectExpr(columnList)
@@ -299,9 +411,10 @@ def schemaDiff(df1, df2):
         1. Make sure that the first dataframe has got more columns than the second
     """
     df1_columns = df1.columns
+    df2_columns_lower = [col.lower() for col in df2.columns]
 
     for columnName in df1.columns:
-        if columnName in df2.columns:
+        if columnName.lower() in df2_columns_lower:
             df1_columns.remove(columnName)
 
     df = df1.select(df1_columns)
@@ -418,3 +531,48 @@ def buildJoinCondition(keyColsList):
         condition = condition + "t.`" + column + "` <=> " + "s.`" + column + "` AND "
 
     return condition[:-5]
+
+
+# A Function for moving files from DSP to Bronze for the given source table. 
+# Since, DSP Writes into two Folders "initial" and "delta", .parquet files from these two folders are recursively
+# moved into Bronze Table Location.
+def move_dsp_to_bronze(srcDatabaseName, srcTableName, src_dsp_path, src_table_path):
+    dsp_path = src_dsp_path
+    bronze_path = src_table_path
+    num_files = 0
+
+    print(f"Recursively copying all files from {dsp_path} to {bronze_path} (flattened, no hierarchy, source not deleted)")
+
+    def list_all_files(path):
+        files = []
+        for item in dbutils.fs.ls(path):
+            if item.isDir():
+                files.extend(list_all_files(item.path))
+            else:
+                files.append(item.path)
+        return files
+
+    try:
+        all_files = list_all_files(dsp_path)
+        for src_file in all_files:
+            file_name = src_file.split("/")[-1]
+            if ".sap." in file_name:
+                dest_file = f"{bronze_path}/{file_name}"
+                dbutils.fs.cp(src_file, dest_file)
+                #num_files += 1
+            else:
+                dest_file = f"{bronze_path}/{file_name}"
+                dbutils.fs.mv(src_file, dest_file)
+                num_files += 1
+    except Exception as e:
+        print(f"Failed to copy files: {e}")
+
+    if num_files > 0:
+        print(f"Copied {num_files} files from {dsp_path} to {bronze_path}, refreshing table {srcDatabaseName}.{srcTableName}")
+        spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
+        print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
+    else:
+        print(f"No files were copied from {dsp_path} to {bronze_path}, so skipping refresh of {srcDatabaseName}.{srcTableName}")
+
+    # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
+    # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/com/phida/main/SilverMerge.py

@@ -1,8 +1,9 @@
+from datetime import date, timedelta
 from com.phida.main.sparksession import spark, logger
-from com.phida.main.Operations import tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
-    dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
+from com.phida.main.Operations import describe_table_to_schema, move_dsp_to_bronze, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
+    dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff, hasColumn
 from com.phida.main.utils import pathExists, convertStrToList
-from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max
+from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max, lit, when, coalesce
 from pyspark.sql.window import Window
 from delta.tables import DeltaTable
 from pyspark.sql import DataFrame
@@ -27,8 +28,11 @@ class SilverMerge:
         cluster_table_path: String - Path to the cluster table
         enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
         use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
+        is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
+        src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
 
     methods:
+        move_dsp_to_bronze
         silverCleansing
         prepareTarget
         upsertToDelta
@@ -38,12 +42,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, is_dsp_kernel, src_dsp_path)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None, is_dsp_kernel='N', src_dsp_path=None):
         """
         desc:
             Initialize the required class variables
@@ -63,9 +67,13 @@ class SilverMerge:
             cluster_table_path: String - Path to the cluster table
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
+            is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
+            src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
         """
 
         logger.info("phida_log: Initialising class variables")
+        logger.info("Reading Bronze Table...")
+        print(f"[SilverMerge] Initialising pipeline for {srcDatabaseName}.{srcTableName} -> {tgtDatabaseName}.{tgtTableName}")
 
         self.srcDatabaseName = srcDatabaseName
         self.srcTableName = srcTableName
@@ -80,34 +88,65 @@ class SilverMerge:
         self.pruneColumn = pruneColumn
         self.cluster_table_path = cluster_table_path
         self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
-        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() == 'Y' else 'N'
+        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() in ('Y') else 'N'
+        self.modified_after = modified_after.strip() if modified_after else (date.today() - timedelta(days=15)).strftime('%Y-%m-%d')
+        self.backfill_interval = backfill_interval if backfill_interval else "1 month"
+        self.is_dsp_kernel = is_dsp_kernel.strip().upper() if is_dsp_kernel else 'N' #To determine if Kernel is loaded via Datasphere
+        self.src_dsp_path = src_dsp_path if src_dsp_path else None
+
+        #Run file movement between DSP and Bronze only if it's a DSP kernel, to ensure we don't have unnecessary file movements for non-DSP kernels
+        if self.is_dsp_kernel == 'Y':
+            src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
+            logger.info(f"phida_log: File movement between DSP and Bronze")
+            print(f"[SilverMerge] File movement between DSP and Bronze")
+            move_dsp_to_bronze(self.srcDatabaseName ,self.srcTableName, self.src_dsp_path, src_table_path)
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
         if tableExists(self.srcDatabaseName, self.srcTableName):
+            logger.info(f"Bronze table {self.srcDatabaseName}.{self.srcTableName} exists.")
+            print(f"[SilverMerge] Bronze table {self.srcDatabaseName}.{self.srcTableName} found.")
 
             logger.info(f"phida_log: source table exists")
 
             logger.info(f"phida_log: initialising derived class variables")
 
             if self.use_auto_loader == 'N':
+                logger.info("Extracting Bronze Structure using Spark readStream.table...")
+                print("[SilverMerge] Reading Bronze table using Spark readStream...")
                 self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+                logger.info("Bronze Structure extracted.")
+                print("[SilverMerge] Bronze structure extracted.")
             elif self.use_auto_loader == 'Y':
+                logger.info("Extracting Bronze Structure using Auto Loader...")
+                print("[SilverMerge] Reading Bronze table using Auto Loader...")
+                desc = spark.sql(f"DESCRIBE TABLE {self.srcDatabaseName}.{self.srcTableName}").collect()
+                schema = describe_table_to_schema(desc)
                 src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
-                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").load(src_table_path).drop("_rescued_data")
+                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").option("modifiedAfter", f"{self.modified_after} 00:00:00.000000 UTC+0").option("cloudFiles.backfillInterval", self.backfill_interval).schema(schema).load(src_table_path).drop("_rescued_data")
+                logger.info("Bronze Structure extracted.")
+                print("[SilverMerge] Bronze structure extracted via Auto Loader.")
 
+            logger.info("Determining key columns for merge...")
+            print("[SilverMerge] Determining key columns for merge...")
             self.keyCols = getKeyCols(self.srcDatabaseName, self.srcTableName)
 
             self.keyColsList = convertStrToList(self.keyCols, ",")
-
+            logger.info(f"Key columns for merge: {self.keyColsList}")
+            print(f"[SilverMerge] Key columns: {self.keyColsList}")
+            
             self.joinCondition = buildJoinCondition(self.keyColsList)
+            logger.info(f"Join condition for merge: {self.joinCondition}")
 
             self.condition = f"{self.pruneColumn} <> '' AND {self.joinCondition}" if self.pruneColumn.strip() \
                              else self.joinCondition
+            logger.info(f"Final merge condition: {self.condition}")
 
             self.dropColumnList = convertStrToList(self.dropColumnStr, ",")
 
             self.columnsDict = buildColumnsDict(self.srcDF, self.dropColumnList)
+            logger.info("Initialization of SilverMerge complete.")
+            print("[SilverMerge] Initialization complete.")
 
     def silverCleansing(self):
         """
@@ -127,21 +166,56 @@ class SilverMerge:
             N/A
         """
         logger.info(f"phida_log: applying cleansing rules on source dataframe ")
+        logger.info("Starting cleansing transformations on Bronze DataFrame...")
+        print("[SilverMerge] Starting Silver cleansing transformations...")
+
 
         if tableExists(self.srcDatabaseName, self.srcTableName):
 
             silverRawDF = self.srcDF
+            logger.info("Bronze DataFrame loaded for cleansing.")
 
             if self.derivedColExpr:
+                logger.info("Adding derived columns to DataFrame...")
                 derivedColExprList = convertStrToList(self.derivedColExpr, "§")
-
                 silverDerivedColumns = addDerivedColumns(silverRawDF, derivedColExprList)
-
+                logger.info("Derived columns added.")
             else:
                 silverDerivedColumns = silverRawDF
+                logger.info("No derived columns specified.")
+
+            # Map DSP __timestamp (timestamp_ntz) to src_commit_time for unified timestamp handling.
+            # And drop __timestamp from the DF
+            # Handle both mixed HVR+DSP and DSP-only inputs safely.
+            has_src_commit_time = "src_commit_time" in silverDerivedColumns.columns
+            has_dsp_timestamp = "__timestamp" in silverDerivedColumns.columns
+
+            if has_dsp_timestamp:
+                if has_src_commit_time:
+                    silverDerivedColumns = silverDerivedColumns.withColumn(
+                        "src_commit_time",
+                        coalesce(col("src_commit_time"), col("__timestamp").cast("timestamp"))
+                    )
+                else:
+                    silverDerivedColumns = silverDerivedColumns.withColumn(
+                        "src_commit_time",
+                        col("__timestamp").cast("timestamp")
+                    )
+
+                silverDerivedColumns = silverDerivedColumns.drop("__timestamp")
+                logger.info("phida_log: Mapped DSP __timestamp to src_commit_time")
+            else:
+                logger.info("phida_log: HVR data; src_commit_time already present")
 
-            self.columnsDict = buildColumnsDict(silverDerivedColumns, self.dropColumnList)
+            # Convert all column names to lower case to ensure
+            # the target Delta table is created and maintained with lower-case column names.
+            silverDerivedColumns = silverDerivedColumns.toDF(*[c.lower() for c in silverDerivedColumns.columns])
+            logger.info("Column names converted to lower case.")
+            print("[SilverMerge] Column names normalised to lower case.")
 
+            self.columnsDict = buildColumnsDict(silverDerivedColumns, self.dropColumnList)
+            logger.info("Cleansing transformations complete.")
+            print("[SilverMerge] Cleansing transformations complete.")
             return silverDerivedColumns
 
     def prepareTarget(self, inDF):
@@ -166,6 +240,7 @@ class SilverMerge:
             N/A
         """
         logger.info(f"phida_log: preparing the target delta table ")
+        print(f"[SilverMerge] Preparing target table {self.tgtDatabaseName}.{self.tgtTableName}...")
 
         targetTableExists = tableExists(self.tgtDatabaseName, self.tgtTableName)
 
@@ -179,6 +254,7 @@ class SilverMerge:
 
             logger.info(f"phida_log: This seems to be the first run")
             logger.info(f"phida_log: creating the target table {self.tgtDatabaseName}.{self.tgtTableName}")
+            print(f"[SilverMerge] First run detected. Creating target table {self.tgtDatabaseName}.{self.tgtTableName}...")
 
             createDeltaTable(inDF,
                              self.tgtTablePath,
@@ -207,6 +283,7 @@ class SilverMerge:
             if addColumns:
                 logger.info(f"phida_log: There seems to be a schema change in silver")
                 logger.info(f"phida_log: Altering the target table {self.tgtDatabaseName}.{self.tgtTableName}")
+                print(f"[SilverMerge] Schema change detected. Altering target table with new columns: {addColumns}")
 
                 alterDeltaTable(self.tgtDatabaseName, self.tgtTableName, addColumns)
 
@@ -214,37 +291,38 @@ class SilverMerge:
 
             else:
                 logger.info(f"phida_log: There is no change in schema in silver")
+                print("[SilverMerge] No schema changes detected.")
 
     def filter_deleted_records(self, microBatchOutputDF):
-           """
-            Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
-            within the last 4 hours based on hvr_integ_key.
-           """
+        """
+        Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
+        within the last 4 hours based on hvr_integ_key.
+        """
            
-           if microBatchOutputDF.isEmpty():
-               return microBatchOutputDF
+        if microBatchOutputDF.isEmpty():
+            return microBatchOutputDF
 
-           filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
+        filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
            
-           if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
-               return microBatchOutputDF
+        if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
+            return microBatchOutputDF
         
-           delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
+        delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
 
-           if delete_records.isEmpty():
-               return microBatchOutputDF
+        if delete_records.isEmpty():
+            return microBatchOutputDF
 
-           delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
+        delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
 
-           joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
+        joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
 
-           filtered_df = (joined_df.filter(
+        filtered_df = (joined_df.filter(
                               (col("delete_hvr_integ_key").isNull()) |
                               (col("hvr_integ_key") >= col("delete_hvr_integ_key"))
                           )
                           .drop("delete_hvr_integ_key"))
 
-           return filtered_df 
+        return filtered_df 
                 
     def preprocess_dataframe(self, df: DataFrame, parent_keys: list) -> DataFrame:
         
@@ -270,8 +348,6 @@ class SilverMerge:
         unpack_keys = set(row["key_columns_for_unpack_table"].split(',')) if row["key_columns_for_unpack_table"] else set()
         parent_keys = list(pack_keys & unpack_keys)
         return parent_keys
-    
-        
 
     def upsertToDelta(self, microBatchOutputDF, batchId):
         """
@@ -291,11 +367,28 @@ class SilverMerge:
         tip:
             N/A
         """
-        op_8_df = microBatchOutputDF.filter("source_operation = 8")
-        microBatchOutputDF = microBatchOutputDF.filter("source_operation in (0,1,2,3)")
-        microBatchOutputDF_8 = microBatchOutputDF.filter("source_operation in (0,1,2,3,8)")
+
+        # DSP_Change: Support both HVR (source_operation) and DSP (__operation_type) operation types
+        # HVR: source_operation in (0,1,2,3) where 0=delete, 1=insert, 2=update, 3=before-image
+        # DSP: __operation_type in ('A','I','L','X') where A=Append, I=Delta-Insert, L=Initial Load, X=delete
+        cols = set(microBatchOutputDF.columns)
+        has_source_operation = "source_operation" in cols
+        has_operation_type = "__operation_type" in cols
+
+        empty_df = microBatchOutputDF.limit(0)
+
+        # Build filtered views once
+        hvr_df_8 = (microBatchOutputDF.filter("source_operation in (0,1,2,3,8)") if has_source_operation else empty_df)
+        op_8_df = (hvr_df_8.filter("source_operation = 8") if has_source_operation else empty_df)
+        hvr_df = (hvr_df_8.filter("source_operation in (0,1,2,3)") if has_source_operation else empty_df)
+        dsp_df = (microBatchOutputDF.filter("__operation_type in ('A','I','L','X')") if has_operation_type else empty_df)
+
+        # Keep op-8/preprocess flow strictly HVR-only. DSP records should not go through op-8 path.
+        microBatchOutputDF_8 = hvr_df_8
+        microBatchOutputDF = hvr_df.unionByName(dsp_df, allowMissingColumns=True)
+
         tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
-        
+
         if self.cluster_table_path:
             if not op_8_df.isEmpty():
                 table_details = DeltaTable.forName(spark, self.cluster_table_path)
@@ -306,9 +399,36 @@ class SilverMerge:
                     tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
                     .whenMatchedDelete() \
                     .execute()
-                microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
-
-        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
+                # Apply op-8 preprocess only to HVR rows, then add DSP rows back.
+                hvr_preprocessed_df = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
+                microBatchOutputDF = hvr_preprocessed_df.unionByName(dsp_df, allowMissingColumns=True)
+
+        # Convert all column names to lower case just before writing to Silver
+        lower_cols = [c.lower() for c in microBatchOutputDF.columns]
+        microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
+
+        # Compatible ordering across all modes:
+        # 1) HVR-only (prod behavior): latest by hvr_integ_key
+        # 2) DSP-only: latest by src_commit_time
+        # 3) Mixed HVR + DSP: DSP rows win for same PK, then latest timestamp
+        batch_cols = set(microBatchOutputDF.columns)
+        has_hvr_key = "hvr_integ_key" in batch_cols
+        has_dsp_op = "__operation_type" in batch_cols
+
+        if has_hvr_key and not has_dsp_op:
+            # HVR-only: keep existing prod behavior
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
+        elif has_hvr_key and has_dsp_op:
+            # Mixed mode: prioritize DSP records, then newest timestamp
+            is_dsp_record = col("hvr_integ_key").isNull()
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(
+                is_dsp_record.desc(),
+                col("src_commit_time").desc(), #__timestamp is casted to src_commit_time in silverCleansing()
+                col("hvr_integ_key").desc()
+            )
+        else:
+            # DSP-only (or no hvr_integ_key): latest timestamp wins
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc()) #__timestamp is casted to src_commit_time in silverCleansing()
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
@@ -316,11 +436,27 @@ class SilverMerge:
         if self.enable_delete_filter == 'Y':
             microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
+        # DSP_Change: Build merge conditions only from columns that actually exist in the batch.
+        # Referencing s.__operation_type when the source is HVR-only causes DELTA_MERGE_UNRESOLVED_EXPRESSION.
+        delete_parts = []
+        if has_source_operation:
+            delete_parts.append("s.source_operation in (0,3)")
+        if has_operation_type:
+            delete_parts.append("s.__operation_type in ('X')")
+
+        if not delete_parts:
+            raise Exception("No operation column found in source batch")
+
+        delete_condition = " OR ".join([f"({p})" for p in delete_parts])
+        update_insert_condition = f"NOT ({delete_condition})"
+
+        print(f"[SilverMerge] Executing merge for batch {batchId} into {self.tgtDatabaseName}.{self.tgtTableName}...")
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
-            .whenMatchedDelete("s.source_operation in (0,3)") \
-            .whenMatchedUpdate(condition="s.source_operation not in (0,3)", set=self.columnsDict) \
-            .whenNotMatchedInsert(condition="s.source_operation not in (0,3)", values=self.columnsDict) \
+            .whenMatchedDelete(delete_condition) \
+            .whenMatchedUpdate(condition=update_insert_condition, set=self.columnsDict) \
+            .whenNotMatchedInsert(condition=update_insert_condition, values=self.columnsDict) \
             .execute()
+        print(f"[SilverMerge] Merge for batch {batchId} complete.")
  
     def streamIntoDeltaTarget(self):
         """
@@ -345,6 +481,7 @@ class SilverMerge:
         self.prepareTarget(silverCleansedDF)
 
         logger.info(f"phida_log: performing streaming merge on target {self.tgtDatabaseName}.{self.tgtTableName}")
+        print(f"[SilverMerge] Starting streaming merge: {self.srcDatabaseName}.{self.srcTableName} -> {self.tgtDatabaseName}.{self.tgtTableName}")
 
         if self.triggerOnce == "Y":
             outputDF = (silverCleansedDF.writeStream
@@ -368,4 +505,4 @@ class SilverMerge:
                         .start(self.tgtTablePath)
                         )
 
-        return outputDF
+        return outputDF

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -96,6 +96,8 @@ class WorkflowBuilder:
             cluster_table_path: String - The path of the table in the cluster
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             enable_autoloader: String - Flag to enable/disable autoloader (default: 'N')
+            modified_after: String - The modified after parameter for autoloader to read only the files modified after the specified timestamp (optional, format: "2026-01-01", "2025-06-01" etc.)
+            backfill_interval: String - The backfill interval for autoloader to read the historical data by looking back a certain interval from the current timestamp (optional, format: "1 month", "10 months", "15 days", "12 hours" etc.)
             pythonWhlPath: String - Path of the python wheel file
             library: String - Library to be installed for the job (with version number)
 
@@ -137,7 +139,9 @@ class WorkflowBuilder:
                     "drop_column_list": dropColumnStr,
                     "cluster_table_path": cluster_table_path,
                     "enable_delete_filter": str(enable_delete_filter),
-                    "enable_autoloader": enable_autoloader
+                    "enable_autoloader": enable_autoloader,
+                    "modified_after": modified_after,
+                    "backfill_interval": backfill_interval
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -348,11 +352,19 @@ class WorkflowBuilder:
                     enable_autoloader = task['enable_autoloader']
                 except:
                     enable_autoloader = ""
+                try:
+                    modified_after = task['modified_after']
+                except:
+                    modified_after = ""
+                try:
+                    backfill_interval = task['backfill_interval']
+                except:
+                    backfill_interval = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.5',
+    version='1.7',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.5 → silverstreamingapplication-1.7}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.5
+Version: 1.7
 Summary: initial version
 Home-page: 
 Author: