SilverStreamingApplication 1.4__tar.gz → 1.6__tar.gz

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.4
+Version: 1.6
 Summary: initial version
 Home-page: 
 Author: 

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/com/phida/main/Operations.py

@@ -1,6 +1,7 @@
 from pyspark.sql.utils import AnalysisException
 from com.phida.main.sparksession import spark
 from pyspark.sql.functions import col
+from pyspark.sql.types import _parse_datatype_string, StructType, StructField
 
 
 def tableExists(dbName, tblName):
@@ -92,6 +93,89 @@ def hiveDDL(df):
 
     return tblDDL
 
+def describe_table_to_schema(desc_rows):
+    """
+    Convert table description rows into a Spark StructType schema.
+    This function parses the output from a DESCRIBE TABLE command and organizes
+    the columns into a StructType schema, separating regular data fields from
+    partition fields.
+    Args:
+        desc_rows: An iterable of row objects containing table metadata.
+                  Each row should have 'col_name' and 'data_type' attributes.
+    Returns:
+        StructType: A Spark StructType object representing the table schema,
+                   with data fields listed first, followed by partition fields.
+    Example:
+        >>> desc_rows = [
+        ...     Row(col_name='id', data_type='bigint'),
+        ...     Row(col_name='name', data_type='string'),
+        ...     Row(col_name='# Partition Information', data_type=''),
+        ...     Row(col_name='date', data_type='date')
+        ... ]
+        >>> schema = describe_table_to_schema(desc_rows)
+    """
+    data_fields = []
+    partition_fields = []
+
+    in_partition_section = False
+
+    for r in desc_rows:
+        col = r.col_name
+        dtype = r.data_type
+
+        # Enter partition section
+        if col.startswith("# Partition Information"):
+            in_partition_section = True
+            continue
+
+        # Skip header rows
+        if col.startswith("#"):
+            continue
+
+        # Skip empty rows (defensive)
+        if not col or not dtype:
+            continue
+
+        field = StructField(
+            col,
+            _parse_datatype(dtype),
+            True
+        )
+
+        if in_partition_section:
+            partition_fields.append(field)
+        else:
+            data_fields.append(field)
+
+    return StructType(data_fields + partition_fields)
+
+def _parse_datatype(dt):
+    """
+    Parse a data type string and return the corresponding Spark data type.
+    
+    This function serves as a wrapper that leverages Spark's internal parsing
+    mechanism to correctly handle complex data types including nested structures,
+    arrays, maps, and other composite types.
+    
+    Args:
+        dt (str): A string representation of a Spark data type 
+                 (e.g., 'IntegerType', 'StructType', 'ArrayType(StringType)')
+    
+    Returns:
+        pyspark.sql.types.DataType: The parsed Spark data type object corresponding
+                                    to the input string.
+    
+    Raises:
+        ParseException: If the data type string is invalid or cannot be parsed.
+    
+    Examples:
+        >>> _parse_datatype('IntegerType')
+        IntegerType
+        >>> _parse_datatype('ArrayType(StringType)')
+        ArrayType(StringType, containsNull=true)
+    """
+    # Let Spark parse complex types correctly
+    return _parse_datatype_string(dt)
 
 def hasColumn(df, columnName):
     """
@@ -299,9 +383,10 @@ def schemaDiff(df1, df2):
         1. Make sure that the first dataframe has got more columns than the second
     """
     df1_columns = df1.columns
+    df2_columns_lower = [col.lower() for col in df2.columns]
 
     for columnName in df1.columns:
-        if columnName in df2.columns:
+        if columnName.lower() in df2_columns_lower:
             df1_columns.remove(columnName)
 
     df = df1.select(df1_columns)

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/com/phida/main/SilverMerge.py

@@ -1,5 +1,6 @@
+from datetime import date, timedelta
 from com.phida.main.sparksession import spark, logger
-from com.phida.main.Operations import tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
+from com.phida.main.Operations import describe_table_to_schema, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff
 from com.phida.main.utils import pathExists, convertStrToList
 from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max
@@ -43,7 +44,7 @@ class SilverMerge:
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None):
         """
         desc:
             Initialize the required class variables
@@ -80,7 +81,9 @@ class SilverMerge:
         self.pruneColumn = pruneColumn
         self.cluster_table_path = cluster_table_path
         self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
-        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() == 'Y' else 'N'
+        self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() in ('Y') else 'N'
+        self.modified_after = modified_after.strip() if modified_after else (date.today() - timedelta(days=15)).strftime('%Y-%m-%d')
+        self.backfill_interval = backfill_interval if backfill_interval else "1 month"
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -93,8 +96,10 @@ class SilverMerge:
             if self.use_auto_loader == 'N':
                 self.srcDF = spark.readStream.table(self.srcDatabaseName + "." + self.srcTableName)
             elif self.use_auto_loader == 'Y':
+                desc = spark.sql(f"DESCRIBE TABLE {self.srcDatabaseName}.{self.srcTableName}").collect()
+                schema = describe_table_to_schema(desc)
                 src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
-                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").load(src_table_path)
+                self.srcDF = spark.readStream.format("cloudFiles").option("cloudFiles.format", "parquet").option("cloudFiles.schemaLocation", src_table_path + "/_cloudfiles/_schemas").option("modifiedAfter", f"{self.modified_after} 00:00:00.000000 UTC+0").option("cloudFiles.backfillInterval", self.backfill_interval).schema(schema).load(src_table_path).drop("_rescued_data")
 
             self.keyCols = getKeyCols(self.srcDatabaseName, self.srcTableName)
 
@@ -240,7 +245,7 @@ class SilverMerge:
 
            filtered_df = (joined_df.filter(
                               (col("delete_hvr_integ_key").isNull()) |
-                              (col("hvr_integ_key") > col("delete_hvr_integ_key"))
+                              (col("hvr_integ_key") >= col("delete_hvr_integ_key"))
                           )
                           .drop("delete_hvr_integ_key"))
 

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/com/phida/main/WorkflowBuilder.py

@@ -72,7 +72,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -96,6 +96,8 @@ class WorkflowBuilder:
             cluster_table_path: String - The path of the table in the cluster
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             enable_autoloader: String - Flag to enable/disable autoloader (default: 'N')
+            modified_after: String - The modified after parameter for autoloader to read only the files modified after the specified timestamp (optional, format: "2026-01-01", "2025-06-01" etc.)
+            backfill_interval: String - The backfill interval for autoloader to read the historical data by looking back a certain interval from the current timestamp (optional, format: "1 month", "10 months", "15 days", "12 hours" etc.)
             pythonWhlPath: String - Path of the python wheel file
             library: String - Library to be installed for the job (with version number)
 
@@ -137,7 +139,9 @@ class WorkflowBuilder:
                     "drop_column_list": dropColumnStr,
                     "cluster_table_path": cluster_table_path,
                     "enable_delete_filter": str(enable_delete_filter),
-                    "enable_autoloader": enable_autoloader
+                    "enable_autoloader": enable_autoloader,
+                    "modified_after": modified_after,
+                    "backfill_interval": backfill_interval
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -348,11 +352,19 @@ class WorkflowBuilder:
                     enable_autoloader = task['enable_autoloader']
                 except:
                     enable_autoloader = ""
+                try:
+                    modified_after = task['modified_after']
+                except:
+                    modified_after = ""
+                try:
+                    backfill_interval = task['backfill_interval']
+                except:
+                    backfill_interval = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.4',
+    version='1.6',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.4 → silverstreamingapplication-1.6}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: silverstreamingapplication
-Version: 1.4
+Version: 1.6
 Summary: initial version
 Home-page: 
 Author: