SilverStreamingApplication 1.9__tar.gz → 1.10__tar.gz

{silverstreamingapplication-1.9 → silverstreamingapplication-1.10}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: silverstreamingapplication
-Version: 1.9
+Version: 1.10
 Summary: initial version
 Home-page: 
 Author: 
 Author-email: 
+Dynamic: summary

{silverstreamingapplication-1.9 → silverstreamingapplication-1.10}/com/phida/main/WorkflowBuilder.py

@@ -60,7 +60,9 @@ class WorkflowBuilder:
              ('dbr_version', ['10.4.x-scala2.12', '10.4.x-scala2.12']),
              ('driver_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
              ('worker_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
-             ('num_workers', ['4', '4'])])
+             # ('num_workers', ['4', '4'])])
+             ('min_workers', ['1', '1']),
+             ('max_workers', ['4', '4'])])
             # example of library argument: "SilverStreamingApplication==1.0"
 
 
@@ -160,7 +162,7 @@ class WorkflowBuilder:
         return taskDict
 
     @staticmethod
-    def createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, numWorkers):
+    def createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, autoscaleFlag, minWorkers, maxWorkers):    
         """
         desc:
             A method for creating the reusable job cluster
@@ -169,7 +171,9 @@ class WorkflowBuilder:
             jobClusterKey - a key for uniquely identifying a job cluster
             DBRVersion - the version of the databricks release to be used
             clusterPolicyId - Define the Id for the cluster policy for which table will be created in JSON
-            numWorkers - The target database for writing the table or creating the metadata
+            autoscaleFlag - 'Y' for an autoscaling cluster, 'N' for a fixed size cluster
+            minWorkers - Minimum workers when autoscaling (ignored when autoscaleFlag is N)
+            maxWorkers - Maximum workers when autoscaling, or the fixed worker count when autoscaleFlag is N
             
 
         return:
@@ -179,20 +183,34 @@ class WorkflowBuilder:
             createTask("task1", "job_cluster_1", "/users/usr/notebook", "/opt/db","<source database name>", "<target database name>", "vbuk")
 
         tip:
-            There are many either or parameters in this
-            Either, enter numWorkers or autoScale as Y. Do no enter both
-            When autoScale is Y, make sure that minWorkers and maxWorkers are present too
+            Autoscaling requires both minWorkers and maxWorkers.
             Either enter workerType and driverType or driverInstancePoolId and instancePoolId.
             Do not enter values for all 4 parameters mentioned above.
         """
-        jobClusterDict = {
-            "job_cluster_key": jobClusterKey,
-            "new_cluster": {
-                "spark_version": DBRVersion,
-                "policy_id": clusterPolicyId,
-                "num_workers": numWorkers
+        
+        if str(autoscaleFlag).strip().upper() == "N":
+            numWorkers = int(maxWorkers)
+            jobClusterDict = {
+                "job_cluster_key": jobClusterKey,
+                "new_cluster": {
+                    "spark_version": DBRVersion,
+                    "policy_id": clusterPolicyId,
+                    "num_workers": numWorkers
+                }
+            }
+        else:
+            minWorkers = int(minWorkers or 1)
+            maxWorkers = int(maxWorkers)
+            if minWorkers > maxWorkers:
+                raise ValueError(f"{jobClusterKey}: min_workers cannot be greater than max_workers")
+            jobClusterDict = {
+                "job_cluster_key": jobClusterKey,
+                "new_cluster": {
+                    "spark_version": DBRVersion,
+                    "policy_id": clusterPolicyId,
+                    "autoscale": {"min_workers": minWorkers, "max_workers": maxWorkers}
+                }
             }
-         }
 
         return jobClusterDict
 
@@ -293,17 +311,21 @@ class WorkflowBuilder:
             for jobCluster in jobClusterList:
                 jobClusterKey = jobCluster['job_cluster_key']
                 DBRVersion = jobCluster['dbr_version']
-                try:
-                    numWorkers = jobCluster['num_workers']
-                except:
-                    numWorkers = None
-                    
                 try:
                     clusterPolicyId = jobCluster['policy_id']
                 except:
                     clusterPolicyId = None
-
-                jobClusterTemp = self.createJobCluster(jobClusterKey, DBRVersion, clusterPolicyId, numWorkers)
+                autoscaleFlag = jobCluster.get('autoscaleFlag', jobCluster.get('autoscale_flag'))
+                minWorkers = jobCluster['min_workers']
+                maxWorkers = jobCluster['max_workers']
+                jobClusterTemp = self.createJobCluster(
+                    jobClusterKey,
+                    DBRVersion,
+                    clusterPolicyId,
+                    autoscaleFlag,
+                    minWorkers,
+                    maxWorkers 
+                )
 
                 jobClusters.append(jobClusterTemp)
 

{silverstreamingapplication-1.9 → silverstreamingapplication-1.10}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='silverstreamingapplication',
-    version='1.9',
+    version='1.10',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',

{silverstreamingapplication-1.9 → silverstreamingapplication-1.10}/silverstreamingapplication.egg-info/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: silverstreamingapplication
-Version: 1.9
+Version: 1.10
 Summary: initial version
 Home-page: 
 Author: 
 Author-email: 
+Dynamic: summary