SilverStreamingApplication 1.8__tar.gz → 1.10__tar.gz

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: silverstreamingapplication
-Version: 1.8
+Version: 1.10
 Summary: initial version
 Home-page: 
 Author: 
 Author-email: 
+Dynamic: summary

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/com/phida/main/Operations.py

@@ -2,10 +2,6 @@ from pyspark.sql.utils import AnalysisException
 from com.phida.main.sparksession import spark
 from pyspark.sql.functions import col
 from pyspark.sql.types import _parse_datatype_string, StructType, StructField
-#Importing dbutils as it is used in file movement.
-from pyspark.dbutils import DBUtils
-
-dbutils = DBUtils(spark.sparkContext)
 
 
 def tableExists(dbName, tblName):
@@ -57,30 +53,14 @@ def getKeyCols(dbName, tblName):
     """
 
     try:
-        # To accomodate both the hvr and dsp data together and seperately
-        # To achive Below Logic:
-        #If Both HVR and DSP -> DSP Wins -> __timestamp
-        #IF Only HVR Or DSP -> Particluar wins
-        table = spark.read.table(f"{dbName}.{tblName}")
-        existing_cols = table.columns
-
-        # Determine the best column to order by
-        order_cols = [c for c in ["src_commit_time", "__timestamp"] if c in existing_cols]
-
-        if order_cols:
-            keyCols = (table
-                        .orderBy(*order_cols, ascending=False)
-                        .select("src_key_cols")
-                        .limit(1)
-                        .collect()[0][0]
-                    )
-        else:
-            keyCols = (table
+        keyCols = (spark.read.table(dbName + "." + tblName)
                         .select("src_key_cols")
                         .limit(1)
                         .collect()[0][0]
                     )
         return keyCols
+           
+ 
     except Exception:
         raise Exception(
             f"the column src_key_cols is not present in the given table {dbName}.{tblName}")
@@ -530,49 +510,4 @@ def buildJoinCondition(keyColsList):
     for column in keyColsList:
         condition = condition + "t.`" + column + "` <=> " + "s.`" + column + "` AND "
 
-    return condition[:-5]
-
-
-# A Function for moving files from DSP to Bronze for the given source table. 
-# Since, DSP Writes into two Folders "initial" and "delta", .parquet files from these two folders are recursively
-# moved into Bronze Table Location.
-def move_dsp_to_bronze(srcDatabaseName, srcTableName, src_dsp_path, src_table_path):
-    dsp_path = src_dsp_path
-    bronze_path = src_table_path
-    num_files = 0
-
-    print(f"Recursively copying all files from {dsp_path} to {bronze_path} (flattened, no hierarchy, source not deleted)")
-
-    def list_all_files(path):
-        files = []
-        for item in dbutils.fs.ls(path):
-            if item.isDir():
-                files.extend(list_all_files(item.path))
-            else:
-                files.append(item.path)
-        return files
-
-    try:
-        all_files = list_all_files(dsp_path)
-        for src_file in all_files:
-            file_name = src_file.split("/")[-1]
-            if ".sap." in file_name:
-                dest_file = f"{bronze_path}/{file_name}"
-                dbutils.fs.cp(src_file, dest_file)
-                #num_files += 1
-            else:
-                dest_file = f"{bronze_path}/{file_name}"
-                dbutils.fs.mv(src_file, dest_file)
-                num_files += 1
-    except Exception as e:
-        print(f"Failed to copy files: {e}")
-
-    if num_files > 0:
-        print(f"Copied {num_files} files from {dsp_path} to {bronze_path}, refreshing table {srcDatabaseName}.{srcTableName}")
-        spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
-        print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
-    else:
-        print(f"No files were copied from {dsp_path} to {bronze_path}, so skipping refresh of {srcDatabaseName}.{srcTableName}")
-
-        # spark.sql(f"REFRESH TABLE {srcDatabaseName}.{srcTableName}")
-        # print(f"Table {srcDatabaseName}.{srcTableName} is Refreshed")
+    return condition[:-5]

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/com/phida/main/SilverMerge.py

@@ -1,6 +1,6 @@
 from datetime import date, timedelta
 from com.phida.main.sparksession import spark, logger
-from com.phida.main.Operations import describe_table_to_schema, move_dsp_to_bronze, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
+from com.phida.main.Operations import describe_table_to_schema, tableExists, addDerivedColumns, createDeltaTable, alterDeltaTable, \
     dropColumns, getDerivedColumnsList, schemaDiff, getKeyCols, buildJoinCondition, buildColumnsDict, hiveDDL, schemaDataTypeDiff, hasColumn
 from com.phida.main.utils import pathExists, convertStrToList
 from pyspark.sql.functions import row_number, col, broadcast,expr, max as spark_max, lit, when, coalesce
@@ -28,11 +28,8 @@ class SilverMerge:
         cluster_table_path: String - Path to the cluster table
         enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
         use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
-        is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
-        src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
 
     methods:
-        move_dsp_to_bronze
         silverCleansing
         prepareTarget
         upsertToDelta
@@ -42,12 +39,12 @@ class SilverMerge:
         from com.phida.SilverMerge import SilverMerge
         silverMergeObj = SilverMerge(srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, is_dsp_kernel, src_dsp_path)
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader)
 
     """
     def __init__(self, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                  tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr,
-                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None, is_dsp_kernel='N', src_dsp_path=None):
+                 triggerOnce, dropColumnStr, pruneColumn, cluster_table_path, enable_delete_filter, use_auto_loader, modified_after=None, backfill_interval=None):
         """
         desc:
             Initialize the required class variables
@@ -67,8 +64,6 @@ class SilverMerge:
             cluster_table_path: String - Path to the cluster table
             enable_delete_filter: String - Flag to enable/disable filtering of deleted records (default: 'N')
             use_auto_loader: String - Flag to enable/disable use of autoloader for reading source table (default: 'N')
-            is_dsp_kernel: String - Flag to indicate if the kernel is a DSP kernel, which requires file movement from DSP to Bronze (default: 'N')
-            src_dsp_path: String - The DSP path to read from if it's a DSP kernel (optional, only needed if is_dsp_kernel is 'Y')
         """
 
         logger.info("phida_log: Initialising class variables")
@@ -90,16 +85,7 @@ class SilverMerge:
         self.enable_delete_filter = enable_delete_filter.strip().upper() if enable_delete_filter else 'N'
         self.use_auto_loader = use_auto_loader.strip().upper() if use_auto_loader and use_auto_loader.strip().upper() in ('Y') else 'N'
         self.modified_after = modified_after.strip() if modified_after else (date.today() - timedelta(days=15)).strftime('%Y-%m-%d')
-        self.backfill_interval = backfill_interval if backfill_interval else "1 month"
-        self.is_dsp_kernel = is_dsp_kernel.strip().upper() if is_dsp_kernel else 'N' #To determine if Kernel is loaded via Datasphere
-        self.src_dsp_path = src_dsp_path if src_dsp_path else None
-
-        #Run file movement between DSP and Bronze only if it's a DSP kernel, to ensure we don't have unnecessary file movements for non-DSP kernels
-        if self.is_dsp_kernel == 'Y':
-            src_table_path = spark.sql(f"DESCRIBE DETAIL {self.srcDatabaseName}.{self.srcTableName}").collect()[0]['location']
-            logger.info(f"phida_log: File movement between DSP and Bronze")
-            print(f"[SilverMerge] File movement between DSP and Bronze")
-            move_dsp_to_bronze(self.srcDatabaseName ,self.srcTableName, self.src_dsp_path, src_table_path)
+        self.backfill_interval = backfill_interval if backfill_interval else "1 week"
 
         logger.info(f"phida_log: Check if source table {self.srcDatabaseName}.{self.srcTableName} exists")
 
@@ -204,8 +190,10 @@ class SilverMerge:
 
                 silverDerivedColumns = silverDerivedColumns.drop("__timestamp")
                 logger.info("phida_log: Mapped DSP __timestamp to src_commit_time")
+                print("[SilverMerge] Mapped DSP __timestamp to src_commit_time")
             else:
                 logger.info("phida_log: HVR data; src_commit_time already present")
+                print("[SilverMerge] HVR data; src_commit_time already present")
 
             # Convert all column names to lower case to ensure
             # the target Delta table is created and maintained with lower-case column names.
@@ -404,8 +392,9 @@ class SilverMerge:
                 microBatchOutputDF = hvr_preprocessed_df.unionByName(dsp_df, allowMissingColumns=True)
 
         # Convert all column names to lower case just before writing to Silver
-        lower_cols = [c.lower() for c in microBatchOutputDF.columns]
-        microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
+        #lower_cols = [c.lower() for c in microBatchOutputDF.columns]
+        #microBatchOutputDF = microBatchOutputDF.toDF(*lower_cols)
+        "Commenting above column lowering as it is already done in silverCleansing() method"
 
         # Compatible ordering across all modes.
         # Do not infer mode from column presence because unionByName(allowMissingColumns=True)
@@ -419,7 +408,7 @@ class SilverMerge:
 
         if has_hvr_key and not has_dsp_op:
             # HVR-only table (no DSP column at all): deduplicate by integ_key descending — highest key = latest change.
-            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc_nulls_last())
+            windowSpec = Window.partitionBy(self.keyColsList).orderBy(col("hvr_integ_key").desc())
         else:
             # Mixed or DSP path: build ordering dynamically based on which recency columns actually exist.
             order_exprs = []

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/com/phida/main/WorkflowBuilder.py

@@ -60,7 +60,9 @@ class WorkflowBuilder:
              ('dbr_version', ['10.4.x-scala2.12', '10.4.x-scala2.12']),
              ('driver_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
              ('worker_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
-             ('num_workers', ['4', '4'])])
+             # ('num_workers', ['4', '4'])])
+             ('min_workers', ['1', '1']),
+             ('max_workers', ['4', '4'])])
             # example of library argument: "SilverStreamingApplication==1.0"
 
 
@@ -72,7 +74,7 @@ class WorkflowBuilder:
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, library):
+                   pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader, modified_after, backfill_interval, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -141,9 +143,7 @@ class WorkflowBuilder:
                     "enable_delete_filter": str(enable_delete_filter),
                     "enable_autoloader": enable_autoloader,
                     "modified_after": modified_after,
-                    "backfill_interval": backfill_interval,
-                    "is_dsp_kernel": is_dsp_kernel,
-                    "src_dsp_path": src_dsp_path
+                    "backfill_interval": backfill_interval
                 }
             },
             "timeout_seconds": timeout_seconds,
@@ -162,7 +162,7 @@ class WorkflowBuilder:
         return taskDict
 
     @staticmethod
-    def createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, numWorkers):
+    def createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, autoscaleFlag, minWorkers, maxWorkers):    
         """
         desc:
             A method for creating the reusable job cluster
@@ -171,7 +171,9 @@ class WorkflowBuilder:
             jobClusterKey - a key for uniquely identifying a job cluster
             DBRVersion - the version of the databricks release to be used
             clusterPolicyId - Define the Id for the cluster policy for which table will be created in JSON
-            numWorkers - The target database for writing the table or creating the metadata
+            autoscaleFlag - 'Y' for an autoscaling cluster, 'N' for a fixed size cluster
+            minWorkers - Minimum workers when autoscaling (ignored when autoscaleFlag is N)
+            maxWorkers - Maximum workers when autoscaling, or the fixed worker count when autoscaleFlag is N
             
 
         return:
@@ -181,20 +183,34 @@ class WorkflowBuilder:
             createTask("task1", "job_cluster_1", "/users/usr/notebook", "/opt/db","<source database name>", "<target database name>", "vbuk")
 
         tip:
-            There are many either or parameters in this
-            Either, enter numWorkers or autoScale as Y. Do no enter both
-            When autoScale is Y, make sure that minWorkers and maxWorkers are present too
+            Autoscaling requires both minWorkers and maxWorkers.
             Either enter workerType and driverType or driverInstancePoolId and instancePoolId.
             Do not enter values for all 4 parameters mentioned above.
         """
-        jobClusterDict = {
-            "job_cluster_key": jobClusterKey,
-            "new_cluster": {
-                "spark_version": DBRVersion,
-                "policy_id": clusterPolicyId,
-                "num_workers": numWorkers
+        
+        if str(autoscaleFlag).strip().upper() == "N":
+            numWorkers = int(maxWorkers)
+            jobClusterDict = {
+                "job_cluster_key": jobClusterKey,
+                "new_cluster": {
+                    "spark_version": DBRVersion,
+                    "policy_id": clusterPolicyId,
+                    "num_workers": numWorkers
+                }
+            }
+        else:
+            minWorkers = int(minWorkers or 1)
+            maxWorkers = int(maxWorkers)
+            if minWorkers > maxWorkers:
+                raise ValueError(f"{jobClusterKey}: min_workers cannot be greater than max_workers")
+            jobClusterDict = {
+                "job_cluster_key": jobClusterKey,
+                "new_cluster": {
+                    "spark_version": DBRVersion,
+                    "policy_id": clusterPolicyId,
+                    "autoscale": {"min_workers": minWorkers, "max_workers": maxWorkers}
+                }
             }
-         }
 
         return jobClusterDict
 
@@ -295,17 +311,21 @@ class WorkflowBuilder:
             for jobCluster in jobClusterList:
                 jobClusterKey = jobCluster['job_cluster_key']
                 DBRVersion = jobCluster['dbr_version']
-                try:
-                    numWorkers = jobCluster['num_workers']
-                except:
-                    numWorkers = None
-                    
                 try:
                     clusterPolicyId = jobCluster['policy_id']
                 except:
                     clusterPolicyId = None
-
-                jobClusterTemp = self.createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, numWorkers)
+                autoscaleFlag = jobCluster.get('autoscaleFlag', jobCluster.get('autoscale_flag'))
+                minWorkers = jobCluster['min_workers']
+                maxWorkers = jobCluster['max_workers']
+                jobClusterTemp = self.createJobCluster(
+                    jobClusterKey,
+                    DBRVersion,
+                    clusterPolicyId,
+                    autoscaleFlag,
+                    minWorkers,
+                    maxWorkers 
+                )
 
                 jobClusters.append(jobClusterTemp)
 
@@ -362,19 +382,11 @@ class WorkflowBuilder:
                     backfill_interval = task['backfill_interval']
                 except:
                     backfill_interval = ""
-                try:
-                    is_dsp_kernel = task['is_dsp_kernel']
-                except:
-                    is_dsp_kernel = ""
-                try:
-                    src_dsp_path = task['src_dsp_path']
-                except:
-                    src_dsp_path = ""
 
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, is_dsp_kernel, src_dsp_path, self.library)
+                                           pruneColumn, cluster_table_path, enable_delete_filter, enable_autoloader,  modified_after, backfill_interval, self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.8',
+    version='1.10',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.8 → silverstreamingapplication-1.10}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: silverstreamingapplication
-Version: 1.8
+Version: 1.10
 Summary: initial version
 Home-page: 
 Author: 
 Author-email: 
+Dynamic: summary