SilverStreamingApplication 1.3__tar.gz → 1.5__tar.gz

silverstreamingapplication-1.5/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: silverstreamingapplication
+Version: 1.5
+Summary: initial version
+Home-page: 
+Author: 
+Author-email: 

{SilverStreamingApplication-1.3 → silverstreamingapplication-1.5}/com/phida/main/SilverMerge.py

@@ -2,7 +2,7 @@ from com.phida.main.sparksession import spark, logger
 from com.phida.main.Operations import tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
 from com.phida.main.utils import pathExists, convertStrToList
-from pyspark.sql.functions import row_number, col, broadcast,expr
+from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max
 from pyspark.sql.window import Window
 from delta.tables import DeltaTable
 from pyspark.sql import DataFrame
@@ -25,6 +25,8 @@ class SilverMerge:
         pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                               (to improve performance of the merge)
         cluster_table_path: String - Path to the cluster table
+        enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+        use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
 
     methods:
         silverCleansing
@@ -36,12 +38,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader):
         """
         desc:
             Initialize the required class variables
@@ -59,6 +61,8 @@ class SilverMerge:
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column to use for dynamic file pruning (future feature)
             cluster_table_path: String - Path to the cluster table
+            enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+            use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
         """
 
         logger.info("phida_log: Initialising class variables")
@@ -75,6 +79,8 @@ class SilverMerge:
         self.dropColumnStr = dropColumnStr
         self.pruneColumn = pruneColumn
         self.cluster_table_path = cluster_table_path
+        self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
+        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() == 'Y' else 'N'
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -84,7 +90,11 @@ class SilverMerge:
 
             logger.info(f"phida_log: initialising derived class variables")
 
-            self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+            if self.use_auto_loader == 'N':
+                self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
+            elif self.use_auto_loader == 'Y':
+                src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
+                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").load(src_table_path).drop("_rescued_data")
 
             self.keyCols = getKeyCols(self.srcDatabaseName, self.srcTableName)
 
@@ -204,7 +214,37 @@ class SilverMerge:
 
             else:
                 logger.info(f"phida_log: There is no change in schema in silver")
-                
+
+    def filter_deleted_records(self, microBatchOutputDF):
+           """
+            Filters out records from microBatchOutputDF that have been marked as deleted in the Bronze table
+            within the last 4 hours based on hvr_integ_key.
+           """
+           
+           if microBatchOutputDF.isEmpty():
+               return microBatchOutputDF
+
+           filtered_table_name = f"{self.srcDatabaseName}.{self.srcTableName}_filtered"
+           
+           if not tableExists(self.srcDatabaseName, f"{self.srcTableName}_filtered"):
+               return microBatchOutputDF
+        
+           delete_records = spark.read.table(filtered_table_name).filter(col("source_operation") == 0)
+
+           if delete_records.isEmpty():
+               return microBatchOutputDF
+
+           delete_keys_df = (delete_records.groupBy(self.keyColsList).agg(spark_max(col("hvr_integ_key")).alias("delete_hvr_integ_key")))
+
+           joined_df = microBatchOutputDF.join(broadcast(delete_keys_df), on=self.keyColsList, how="left")
+
+           filtered_df = (joined_df.filter(
+                              (col("delete_hvr_integ_key").isNull()) |
+                              (col("hvr_integ_key") >= col("delete_hvr_integ_key"))
+                          )
+                          .drop("delete_hvr_integ_key"))
+
+           return filtered_df 
                 
     def preprocess_dataframe(self, df: DataFrame, parent_keys: list) -> DataFrame:
         
@@ -256,23 +296,25 @@ class SilverMerge:
         microBatchOutputDF_8 = microBatchOutputDF.filter("source_operation in (0,1,2,3,8)")
         tgtDeltaTable = DeltaTable.forName(spark, self.tgtDatabaseName + "." + self.tgtTableName)
         
-        if not op_8_df.isEmpty():
-            table_details = DeltaTable.forName(spark,self.cluster_table_path)
-            table_details_df = table_details.toDF()
-            parent_keys = self.get_parent_keys(table_details_df, self.srcDatabaseName, self.srcTableName)
-            parent_key_conditions = " AND ".join([f"t.{col} = s.{col}" for col in parent_keys])
-            if not tgtDeltaTable.toDF().isEmpty():
-                tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
+        if self.cluster_table_path:
+            if not op_8_df.isEmpty():
+                table_details = DeltaTable.forName(spark, self.cluster_table_path)
+                table_details_df = table_details.toDF()
+                parent_keys = self.get_parent_keys(table_details_df, self.srcDatabaseName, self.srcTableName)
+                parent_key_conditions = " AND ".join([f"t.{col} = s.{col}" for col in parent_keys])
+                if not tgtDeltaTable.toDF().isEmpty():
+                    tgtDeltaTable.alias("t").merge(microBatchOutputDF_8.alias("s"), parent_key_conditions) \
                     .whenMatchedDelete() \
                     .execute()
-            microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
-            
-        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("src_commit_time").desc(),
-                                                                  col("hvr_integ_key").desc())
+                microBatchOutputDF = self.preprocess_dataframe(microBatchOutputDF_8, parent_keys)
+
+        windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
 
         microBatchOutputDF = microBatchOutputDF.withColumn("latest_record", row_number().over(windowSpec)).filter(
             "latest_record == 1").drop("latest_record")
 
+        if self.enable_delete_filter == 'Y':
+            microBatchOutputDF = self.filter_deleted_records(microBatchOutputDF)
 
         tgtDeltaTable.alias("t").merge(microBatchOutputDF.alias("s"), self.condition) \
             .whenMatchedDelete("s.source_operation in (0,3)") \

{SilverStreamingApplication-1.3 → silverstreamingapplication-1.5}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn,cluster_table_path, pythonWhlPath, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -94,6 +94,8 @@ class WorkflowBuilder:
             pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                                   (to improve performance of the merge)
             cluster_table_path: String - The path of the table in the cluster
+            enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
+            enable_autoloader: String - Flag to enable/disable autoloader (default: 'N')
             pythonWhlPath: String - Path of the python wheel file
             library: String - Library to be installed for the job (with version number)
 
@@ -133,7 +135,9 @@ class WorkflowBuilder:
                     "trigger_once": triggerOnce,
                     "prune_column": pruneColumn,
                     "drop_column_list": dropColumnStr,
-                    "cluster_table_path": cluster_table_path
+                    "cluster_table_path": cluster_table_path,
+                    "enable_delete_filter": str(enable_delete_filter),
+                    "enable_autoloader": enable_autoloader
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -334,11 +338,21 @@ class WorkflowBuilder:
                     cluster_table_path = task['cluster_table_path']
                 except:
                     cluster_table_path = ""
+                    
+                try:
+                    enable_delete_filter = task['enable_delete_filter']
+                except:
+                    enable_delete_filter = ""
+
+                try:
+                    enable_autoloader = task['enable_autoloader']
+                except:
+                    enable_autoloader = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn,cluster_table_path, "", self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{SilverStreamingApplication-1.3 → silverstreamingapplication-1.5}/setup.py

@@ -1,8 +1,8 @@
 from setuptools import setup
 
 setup(
-    name='SilverStreamingApplication',
-    version='1.3',
+    name='silverstreamingapplication',
+    version='1.5',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

silverstreamingapplication-1.5/silverstreamingapplication.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: silverstreamingapplication
+Version: 1.5
+Summary: initial version
+Home-page: 
+Author: 
+Author-email: 

{SilverStreamingApplication-1.3/SilverStreamingApplication.egg-info → silverstreamingapplication-1.5/silverstreamingapplication.egg-info}/SOURCES.txt

@@ -1,10 +1,4 @@
 setup.py
-SilverStreamingApplication.egg-info/PKG-INFO
-SilverStreamingApplication.egg-info/SOURCES.txt
-SilverStreamingApplication.egg-info/dependency_links.txt
-SilverStreamingApplication.egg-info/top_level.txt
-com/__init__.py
-com/phida/__init__.py
 com/phida/main/Maintenance.py
 com/phida/main/Operations.py
 com/phida/main/PartitionAnalyzer.py
@@ -19,4 +13,8 @@ com/phida/main/utilsDatabricksAPI.py
 com/phida/tests/OperationsTest.py
 com/phida/tests/__init__.py
 com/phida/tests/main_test.py
-com/phida/tests/utils_test.py
+com/phida/tests/utils_test.py
+silverstreamingapplication.egg-info/PKG-INFO
+silverstreamingapplication.egg-info/SOURCES.txt
+silverstreamingapplication.egg-info/dependency_links.txt
+silverstreamingapplication.egg-info/top_level.txt

SilverStreamingApplication-1.3/PKG-INFO

@@ -1,12 +0,0 @@
-Metadata-Version: 2.1
-Name: SilverStreamingApplication
-Version: 1.3
-Summary: initial version
-Home-page: UNKNOWN
-Author: 
-Author-email: 
-License: UNKNOWN
-Platform: UNKNOWN
-
-UNKNOWN
-

SilverStreamingApplication-1.3/SilverStreamingApplication.egg-info/PKG-INFO

@@ -1,12 +0,0 @@
-Metadata-Version: 2.1
-Name: SilverStreamingApplication
-Version: 1.3
-Summary: initial version
-Home-page: UNKNOWN
-Author: 
-Author-email: 
-License: UNKNOWN
-Platform: UNKNOWN
-
-UNKNOWN
-