SilverStreamingApplication 1.2__tar.gz → 1.4__tar.gz

silverstreamingapplication-1.4/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: silverstreamingapplication
+Version: 1.4
+Summary: initial version
+Home-page: 
+Author: 
+Author-email: 

{SilverStreamingApplication-1.2 → silverstreamingapplication-1.4}/com/phida/main/SilverMerge.py

@@ -2,10 +2,10 @@ from com.phida.main.sparksession import spark, logger
 from com.phida.main.Operations import tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
 from com.phida.main.utils import pathExists, convertStrToList
-from pyspark.sql.functions import row_number, col, broadcast
+from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max
 from pyspark.sql.window import Window
 from delta.tables import DeltaTable
-
+from pyspark.sql import DataFrame
 
 class SilverMerge:
     """
@@ -24,6 +24,9 @@ class SilverMerge:
         dropColumnStr: String - Columns to be dropped from source table df
         pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                               (to improve performance of the merge)
+        cluster_table_path: String - Path to the cluster table
+        enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+        use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
 
     methods:
         silverCleansing
@@ -35,12 +38,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader):
         """
         desc:
             Initialize the required class variables
@@ -57,7 +60,9 @@ class SilverMerge:
             triggerOnce: String - Whether continuous streaming or just once
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column to use for dynamic file pruning (future feature)
-
+            cluster_table_path: String - Path to the cluster table
+            enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+            use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
         """
 
         logger.info("phida_log: Initialising class variables")
@@ -73,6 +78,9 @@ class SilverMerge:
         self.triggerOnce = triggerOnce
         self.dropColumnStr = dropColumnStr
         self.pruneColumn = pruneColumn
+        self.cluster_table_path = cluster_table_path
+        self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
+        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() == 'Y' else 'N'
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -82,7 +90,11 @@ class SilverMerge:
 
             logger.info(f"phida_log: initialising derived class variables")
 
-            self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+            if self.use_auto_loader == 'N':
+                self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+            elif self.use_auto_loader == 'Y':
+                src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
+                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").load(src_table_path)
 
             self.keyCols = getKeyCols(self.srcDatabaseName, self.srcTableName)
 
@@ -203,6 +215,64 @@ class SilverMerge:
             else:
                 logger.info(f"phida_log: There is no change in schema in silver")
 
+    def filter_deleted_records(self, microBatchOutputDF):
+           """
+            Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
+            within the last 4 hours based on hvr_integ_key.
+           """
+           
+           if microBatchOutputDF.isEmpty():
+               return microBatchOutputDF
+
+           filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
+           
+           if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
+               return microBatchOutputDF
+        
+           delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
+
+           if delete_records.isEmpty():
+               return microBatchOutputDF
+
+           delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
+
+           joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
+
+           filtered_df = (joined_df.filter(
+                              (col("delete_hvr_integ_key").isNull()) |
+                              (col("hvr_integ_key") > col("delete_hvr_integ_key"))
+                          )
+                          .drop("delete_hvr_integ_key"))
+
+           return filtered_df 
+                
+    def preprocess_dataframe(self, df: DataFrame, parent_keys: list) -> DataFrame:
+        
+            """Removes rows where all or any 'source_operation' values for a group are 8."""
+
+            grouped_df = df.groupBy(parent_keys).agg(
+                expr("bool_and(source_operation = 8)").alias("all_8"),
+                expr("bool_or(source_operation = 8)").alias("any_8")
+            )
+
+            return df.join(grouped_df, on=parent_keys, how="left").filter("NOT (all_8 OR any_8)")
+        
+    
+    def get_parent_keys(self, df: DataFrame, kernel_value: str, unpack_table_value: str) -> list:
+        
+        """Get the parent keys for the given kernel and unpack_table values"""
+        
+        filtered_df = df.filter(
+            (col("kernel") == kernel_value) & (col("unpack_table") == unpack_table_value)
+        )
+        row = filtered_df.select("key_columns_for_pack_table", "key_columns_for_unpack_table").first()
+        pack_keys = set(row["key_columns_for_pack_table"].split(',')) if row["key_columns_for_pack_table"] else set()
+        unpack_keys = set(row["key_columns_for_unpack_table"].split(',')) if row["key_columns_for_unpack_table"] else set()
+        parent_keys = list(pack_keys & unpack_keys)
+        return parent_keys
+    
+        
+
     def upsertToDelta(self, microBatchOutputDF, batchId):
         """
         desc:
@@ -221,16 +291,30 @@ class SilverMerge:
         tip:
             N/A
         """
-         
+        op_8_df = microBatchOutputDF.filter("source_operation = 8")
         microBatchOutputDF = microBatchOutputDF.filter("source_operation in (0,1,2,3)")
-
-        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc(),
-                                                                  col("hvr_integ_key").desc())
+        microBatchOutputDF_8 = microBatchOutputDF.filter("source_operation in (0,1,2,3,8)")
+        tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
+        
+        if self.cluster_table_path:
+            if not op_8_df.isEmpty():
+                table_details = DeltaTable.forName(spark, self.cluster_table_path)
+                table_details_df = table_details.toDF()
+                parent_keys = self.get_parent_keys(table_details_df, self.srcDatabaseName, self.srcTableName)
+                parent_key_conditions = " AND ".join([f"t.{col} = s.{col}" for col in parent_keys])
+                if not tgtDeltaTable.toDF().isEmpty():
+                    tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
+                    .whenMatchedDelete() \
+                    .execute()
+                microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
+
+        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
 
-        tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
+        if self.enable_delete_filter == 'Y':
+            microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
             .whenMatchedDelete("s.source_operation in (0,3)") \

{SilverStreamingApplication-1.2 → silverstreamingapplication-1.4}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, pythonWhlPath, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -93,6 +93,10 @@ class WorkflowBuilder:
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                                   (to improve performance of the merge)
+            cluster_table_path: String - The path of the table in the cluster
+            enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+            enable_autoloader: String - Flag to enable/disable autoloader (default: 'N')
+            pythonWhlPath: String - Path of the python wheel file
             library: String - Library to be installed for the job (with version number)
 
         return:
@@ -130,7 +134,10 @@ class WorkflowBuilder:
                     "derived_col_expr": derivedColExpr,
                     "trigger_once": triggerOnce,
                     "prune_column": pruneColumn,
-                    "drop_column_list": dropColumnStr
+                    "drop_column_list": dropColumnStr,
+                    "cluster_table_path": cluster_table_path,
+                    "enable_delete_filter": str(enable_delete_filter),
+                    "enable_autoloader": enable_autoloader
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -311,7 +318,7 @@ class WorkflowBuilder:
                 tgtTablePath = task['tgt_table_path']
                 triggerOnce = task['trigger_once']
                 dropColumnStr = task['drop_column_list']
-
+                
                 try:
                     tgtPartitionColumns = task['tgt_partition_columns']
                 except:
@@ -326,11 +333,26 @@ class WorkflowBuilder:
                     pruneColumn = task['prune_column']
                 except:
                     pruneColumn = ""
+                    
+                try:
+                    cluster_table_path = task['cluster_table_path']
+                except:
+                    cluster_table_path = ""
+                    
+                try:
+                    enable_delete_filter = task['enable_delete_filter']
+                except:
+                    enable_delete_filter = ""
+
+                try:
+                    enable_autoloader = task['enable_autoloader']
+                except:
+                    enable_autoloader = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, "", self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{SilverStreamingApplication-1.2 → silverstreamingapplication-1.4}/setup.py

@@ -1,8 +1,8 @@
 from setuptools import setup
 
 setup(
-    name='SilverStreamingApplication',
-    version='1.2',
+    name='silverstreamingapplication',
+    version='1.4',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

silverstreamingapplication-1.4/silverstreamingapplication.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: silverstreamingapplication
+Version: 1.4
+Summary: initial version
+Home-page: 
+Author: 
+Author-email: 

{SilverStreamingApplication-1.2/SilverStreamingApplication.egg-info → silverstreamingapplication-1.4/silverstreamingapplication.egg-info}/SOURCES.txt

@@ -1,10 +1,4 @@
 setup.py
-SilverStreamingApplication.egg-info/PKG-INFO
-SilverStreamingApplication.egg-info/SOURCES.txt
-SilverStreamingApplication.egg-info/dependency_links.txt
-SilverStreamingApplication.egg-info/top_level.txt
-com/__init__.py
-com/phida/__init__.py
 com/phida/main/Maintenance.py
 com/phida/main/Operations.py
 com/phida/main/PartitionAnalyzer.py
@@ -19,4 +13,8 @@ com/phida/main/utilsDatabricksAPI.py
 com/phida/tests/OperationsTest.py
 com/phida/tests/__init__.py
 com/phida/tests/main_test.py
-com/phida/tests/utils_test.py
+com/phida/tests/utils_test.py
+silverstreamingapplication.egg-info/PKG-INFO
+silverstreamingapplication.egg-info/SOURCES.txt
+silverstreamingapplication.egg-info/dependency_links.txt
+silverstreamingapplication.egg-info/top_level.txt

SilverStreamingApplication-1.2/PKG-INFO

@@ -1,12 +0,0 @@
-Metadata-Version: 2.1
-Name: SilverStreamingApplication
-Version: 1.2
-Summary: initial version
-Home-page: UNKNOWN
-Author: 
-Author-email: 
-License: UNKNOWN
-Platform: UNKNOWN
-
-UNKNOWN
-

SilverStreamingApplication-1.2/SilverStreamingApplication.egg-info/PKG-INFO

@@ -1,12 +0,0 @@
-Metadata-Version: 2.1
-Name: SilverStreamingApplication
-Version: 1.2
-Summary: initial version
-Home-page: UNKNOWN
-Author: 
-Author-email: 
-License: UNKNOWN
-Platform: UNKNOWN
-
-UNKNOWN
-