SilverStreamingApplication 1.2__tar.gz → 1.3__tar.gz

{SilverStreamingApplication-1.2 → SilverStreamingApplication-1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SilverStreamingApplication
-Version: 1.2
+Version: 1.3
 Summary: initial version
 Home-page: UNKNOWN
 Author: 

{SilverStreamingApplication-1.2 → SilverStreamingApplication-1.3}/SilverStreamingApplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SilverStreamingApplication
-Version: 1.2
+Version: 1.3
 Summary: initial version
 Home-page: UNKNOWN
 Author: 

{SilverStreamingApplication-1.2 → SilverStreamingApplication-1.3}/com/phida/main/SilverMerge.py

@@ -2,10 +2,10 @@ from com.phida.main.sparksession import spark, logger
 from com.phida.main.Operations import tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
 from com.phida.main.utils import pathExists, convertStrToList
-from pyspark.sql.functions import row_number, col, broadcast
+from pyspark.sql.functions import row_number, col, broadcast,expr
 from pyspark.sql.window import Window
 from delta.tables import DeltaTable
-
+from pyspark.sql import DataFrame
 
 class SilverMerge:
     """
@@ -24,6 +24,7 @@ class SilverMerge:
         dropColumnStr: String - Columns to be dropped from source table df
         pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                               (to improve performance of the merge)
+        cluster_table_path: String - Path to the cluster table
 
     methods:
         silverCleansing
@@ -35,12 +36,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path):
         """
         desc:
             Initialize the required class variables
@@ -57,7 +58,7 @@ class SilverMerge:
             triggerOnce: String - Whether continuous streaming or just once
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column to use for dynamic file pruning (future feature)
-
+            cluster_table_path: String - Path to the cluster table
         """
 
         logger.info("phida_log: Initialising class variables")
@@ -73,6 +74,7 @@ class SilverMerge:
         self.triggerOnce = triggerOnce
         self.dropColumnStr = dropColumnStr
         self.pruneColumn = pruneColumn
+        self.cluster_table_path = cluster_table_path
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -202,6 +204,34 @@ class SilverMerge:
 
             else:
                 logger.info(f"phida_log: There is no change in schema in silver")
+                
+                
+    def preprocess_dataframe(self, df: DataFrame, parent_keys: list) -> DataFrame:
+        
+            """Removes rows where all or any 'source_operation' values for a group are 8."""
+
+            grouped_df = df.groupBy(parent_keys).agg(
+                expr("bool_and(source_operation = 8)").alias("all_8"),
+                expr("bool_or(source_operation = 8)").alias("any_8")
+            )
+
+            return df.join(grouped_df, on=parent_keys, how="left").filter("NOT (all_8 OR any_8)")
+        
+    
+    def get_parent_keys(self, df: DataFrame, kernel_value: str, unpack_table_value: str) -> list:
+        
+        """Get the parent keys for the given kernel and unpack_table values"""
+        
+        filtered_df = df.filter(
+            (col("kernel") == kernel_value) & (col("unpack_table") == unpack_table_value)
+        )
+        row = filtered_df.select("key_columns_for_pack_table", "key_columns_for_unpack_table").first()
+        pack_keys = set(row["key_columns_for_pack_table"].split(',')) if row["key_columns_for_pack_table"] else set()
+        unpack_keys = set(row["key_columns_for_unpack_table"].split(',')) if row["key_columns_for_unpack_table"] else set()
+        parent_keys = list(pack_keys & unpack_keys)
+        return parent_keys
+    
+        
 
     def upsertToDelta(self, microBatchOutputDF, batchId):
         """
@@ -221,16 +251,28 @@ class SilverMerge:
         tip:
             N/A
         """
-         
+        op_8_df = microBatchOutputDF.filter("source_operation = 8")
         microBatchOutputDF = microBatchOutputDF.filter("source_operation in (0,1,2,3)")
-
+        microBatchOutputDF_8 = microBatchOutputDF.filter("source_operation in (0,1,2,3,8)")
+        tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
+        
+        if not op_8_df.isEmpty():
+            table_details = DeltaTable.forName(spark,self.cluster_table_path)
+            table_details_df = table_details.toDF()
+            parent_keys = self.get_parent_keys(table_details_df, self.srcDatabaseName, self.srcTableName)
+            parent_key_conditions = " AND ".join([f"t.{col} = s.{col}" for col in parent_keys])
+            if not tgtDeltaTable.toDF().isEmpty():
+                tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
+                    .whenMatchedDelete() \
+                    .execute()
+            microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
+            
         windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc(),
                                                                   col("hvr_integ_key").desc())
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
 
-        tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
 
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
             .whenMatchedDelete("s.source_operation in (0,3)") \

{SilverStreamingApplication-1.2 → SilverStreamingApplication-1.3}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, pythonWhlPath, library):
+                   pruneColumn,cluster_table_path, pythonWhlPath, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -93,6 +93,8 @@ class WorkflowBuilder:
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                                   (to improve performance of the merge)
+            cluster_table_path: String - The path of the table in the cluster
+            pythonWhlPath: String - Path of the python wheel file
             library: String - Library to be installed for the job (with version number)
 
         return:
@@ -130,7 +132,8 @@ class WorkflowBuilder:
                     "derived_col_expr": derivedColExpr,
                     "trigger_once": triggerOnce,
                     "prune_column": pruneColumn,
-                    "drop_column_list": dropColumnStr
+                    "drop_column_list": dropColumnStr,
+                    "cluster_table_path": cluster_table_path
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -311,7 +314,7 @@ class WorkflowBuilder:
                 tgtTablePath = task['tgt_table_path']
                 triggerOnce = task['trigger_once']
                 dropColumnStr = task['drop_column_list']
-
+                
                 try:
                     tgtPartitionColumns = task['tgt_partition_columns']
                 except:
@@ -326,11 +329,16 @@ class WorkflowBuilder:
                     pruneColumn = task['prune_column']
                 except:
                     pruneColumn = ""
+                    
+                try:
+                    cluster_table_path = task['cluster_table_path']
+                except:
+                    cluster_table_path = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, "", self.library)
+                                           pruneColumn,cluster_table_path, "", self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{SilverStreamingApplication-1.2 → SilverStreamingApplication-1.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='SilverStreamingApplication',
-    version='1.2',
+    version='1.3',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',