SilverStreamingApplication 1.0__tar.gz → 1.1__tar.gz

{SilverStreamingApplication-1.0 → SilverStreamingApplication-1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SilverStreamingApplication
-Version: 1.0
+Version: 1.1
 Summary: initial version
 Home-page: UNKNOWN
 Author: 

{SilverStreamingApplication-1.0 → SilverStreamingApplication-1.1}/SilverStreamingApplication.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: SilverStreamingApplication
-Version: 1.0
+Version: 1.1
 Summary: initial version
 Home-page: UNKNOWN
 Author: 

{SilverStreamingApplication-1.0 → SilverStreamingApplication-1.1}/com/phida/main/WorkflowBuilder.py

@@ -25,7 +25,7 @@ class WorkflowBuilder:
 
         """
 
-    def __init__(self, inputDict: OrderedDict, onFailureEmailList: list):
+    def __init__(self, inputDict: OrderedDict, onFailureEmailList: list, library: str):
         """"
         desc:
         Initialize the required variables for the class
@@ -34,6 +34,7 @@ class WorkflowBuilder:
             inputDict: inputDict - A single ordered dictionary containing all the possible configurations
              for building the JSON in the databricks jobs API format
             onFailureEmailList: List of string - A list of emails to be notified when a job failed
+            library: string - PyPI library name with version number 
 
         example:
             # Sample data for inputDict argument:
@@ -60,16 +61,18 @@ class WorkflowBuilder:
              ('driver_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
              ('worker_type', ['Standard_E8ds_v4', 'Standard_E8ds_v4']),
              ('num_workers', ['4', '4'])])
+            # example of library argument: "SilverStreamingApplication==1.0"
 
 
         """
         self.inputDict = inputDict
         self.onFailureEmailList = onFailureEmailList
+        self.library = library
 
     @staticmethod
     def createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName, tgtDatabaseName, tgtTableName,
                    tgtCheckpoint, tgtTablePath, tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                   pruneColumn, pythonWhlPath):
+                   pruneColumn, pythonWhlPath, library):
         """
         desc:
             A method for creating the dict/json for a single task
@@ -90,6 +93,7 @@ class WorkflowBuilder:
             dropColumnStr: String - Columns to be dropped from source table df
             pruneColumn: String - Column for applying the prune filter in the merge ON condition clause \
                                   (to improve performance of the merge)
+            library: String - Library to be installed for the job (with version number)
 
         return:
             silverCleansedDF: dict - returns the task in jobs API format as a dictionary
@@ -132,7 +136,14 @@ class WorkflowBuilder:
             "timeout_seconds": timeout_seconds,
             "max_retries": max_retries,
             "min_retry_interval_millis": 900000,
-            "retry_on_timeout": "false"
+            "retry_on_timeout": "false", 
+            "libraries": [
+                {
+                    "pypi": {
+                        "package": library
+                    }
+                }
+            ]
         }
 
         return taskDict
@@ -319,7 +330,7 @@ class WorkflowBuilder:
                 taskTemp = self.createTask(taskKey, jobClusterKey, notebookPath, srcDatabaseName, srcTableName,
                                            tgtDatabaseName, tgtTableName, tgtCheckpoint, tgtTablePath,
                                            tgtPartitionColumns, derivedColExpr, triggerOnce, dropColumnStr,
-                                           pruneColumn, "")
+                                           pruneColumn, "", self.library)
 
                 tasks.append(taskTemp)
             if jobId is None:

{SilverStreamingApplication-1.0 → SilverStreamingApplication-1.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='SilverStreamingApplication',
-    version='1.0',
+    version='1.1',
     packages=['com.phida.main', 'com.phida.tests'],
     url='',
     license='',