zipher-sdk 0.3.0__tar.gz → 0.4.1__tar.gz

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/PKG-INFO

@@ -1,17 +1,25 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: zipher-sdk
-Version: 0.3.0
+Version: 0.4.1
 Summary: Python SDK for programmatic access to the Zipher API
-Home-page: UNKNOWN
 Author: Zipher Inc.
 Author-email: yoav@zipher.cloud
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: pydantic==2.11.3
+Requires-Dist: requests==2.32.3
+Requires-Dist: databricks-sdk==0.52.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
 
 
 # Zipher SDK
@@ -136,4 +144,69 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 
 ```
 
+### Update Existing Multiple Tasks Configuration
 
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
+
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```

zipher-sdk-0.3.0/zipher_sdk.egg-info/PKG-INFO → zipher_sdk-0.4.1/README.md

@@ -1,18 +1,3 @@
-Metadata-Version: 2.1
-Name: zipher-sdk
-Version: 0.3.0
-Summary: Python SDK for programmatic access to the Zipher API
-Home-page: UNKNOWN
-Author: Zipher Inc.
-Author-email: yoav@zipher.cloud
-License: UNKNOWN
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Description-Content-Type: text/markdown
-License-File: LICENSE
-
 
 # Zipher SDK
 
@@ -136,4 +121,69 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 
 ```
 
+### Update Existing Multiple Tasks Configuration
+
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
 
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/setup.py

@@ -8,7 +8,7 @@ with open('README.md', encoding='utf-8') as f:
 
 setup(
     name='zipher-sdk',
-    version='0.3.0',
+    version='0.4.1',
     description='Python SDK for programmatic access to the Zipher API',
     long_description=long_description,
     long_description_content_type='text/markdown',

zipher_sdk-0.4.1/zipher/__init__.py

@@ -0,0 +1,2 @@
+from .client import Client
+from .models import Config

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/zipher/acl/zipher_installer.py

@@ -122,7 +122,7 @@ class ZipherInstaller:
         start_date = datetime.now() - timedelta(days=days_back)
         start_date_ms_int = int(start_date.timestamp() * 1000)
 
-        jobs = self.dbx_client.jobs.list(limit=100)
+        jobs = self.dbx_client.jobs.list()
         runs = self.dbx_client.jobs.list_runs(limit=0, start_time_from=start_date_ms_int)
 
         all_job_ids = set()

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/zipher/client.py

@@ -4,11 +4,9 @@ from typing import Dict, Optional
 
 import requests
 from databricks.sdk.service.compute import ClusterSpec
-
-from zipher.models import ConfFetcherRequest, Config
+from zipher.models import ConfFetcherRequest, GetOptimizedTasksRequest, Config
 from zipher.exceptions import MissingAPIKeyError
 
-
 class Client:
     def __init__(self, customer_id: str, zipher_api_key: str = '', config: Optional[Config] = None):
         self.config = config or Config()
@@ -33,17 +31,35 @@ class Client:
         else:
             response.raise_for_status()
 
-    def get_optimized_config(self, job_id: str) -> Dict:
-        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id)
+    def _call_get_optimized_tasks_api(self, params: GetOptimizedTasksRequest):
+        headers = {
+            'x-api-key': self.zipher_api_key
+        }
+        response = requests.get(
+            url=self.config.zipher_get_optimized_tasks_api_endpoint,
+            headers=headers,
+            params=params.model_dump()
+        )
+        if response.ok:
+            return response.json()
+        else:
+            response.raise_for_status()
+
+    def get_optimized_tasks(self, job_id: str, tasks: list[dict]) -> list[dict]:
+        params = GetOptimizedTasksRequest(customer_id=self.customer_id, job_id=job_id, tasks=json.dumps(tasks))
+        return self._call_get_optimized_tasks_api(params)
+    
+    def get_optimized_config(self, job_id: str, task_key: Optional[str] = None) -> Dict:
+        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, task_key=task_key)
         return self._call_conf_fetcher_api(params)
 
     def get_optimized_config_as_dbx_cluster_spec(self, job_id: str) -> ClusterSpec:
         return ClusterSpec.from_dict(self.get_optimized_config(job_id))
 
-    def update_existing_conf(self, job_id: str, existing_conf: Dict) -> Dict:
+    def update_existing_conf(self, job_id: str, existing_conf: Dict, task_key: Optional[str] = None) -> Dict:
         if 'new_cluster' in existing_conf:
             existing_conf = existing_conf['new_cluster']
-        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, merge_with=json.dumps(existing_conf))
+        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, merge_with=json.dumps(existing_conf), task_key=task_key)
         try:
             return self._call_conf_fetcher_api(params)
         except:

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/zipher/models.py

@@ -12,6 +12,7 @@ class WorkspaceClientAuthType(Enum):
 class ConfFetcherRequest(BaseModel):
     job_id: str
     customer_id: str
+    task_key: Optional[str] = None
     merge_with: Optional[str] = None
 
 
@@ -24,3 +25,9 @@ class Config(BaseModel, extra='forbid', use_enum_values=True):
     databricks_client_secret_env_var: str = 'DATABRICKS_CLIENT_SECRET'
 
     zipher_config_fetcher_api_endpoint: str = 'https://api.zipher.cloud/default/get_clusterspec'
+    zipher_get_optimized_tasks_api_endpoint: str = 'https://api.zipher.cloud/default/get_optimized_tasks'
+
+class GetOptimizedTasksRequest(BaseModel):
+    customer_id: str
+    job_id: str
+    tasks: Optional[str] = None

zipher-sdk-0.3.0/README.md → zipher_sdk-0.4.1/zipher_sdk.egg-info/PKG-INFO

@@ -1,3 +1,26 @@
+Metadata-Version: 2.4
+Name: zipher-sdk
+Version: 0.4.1
+Summary: Python SDK for programmatic access to the Zipher API
+Author: Zipher Inc.
+Author-email: yoav@zipher.cloud
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic==2.11.3
+Requires-Dist: requests==2.32.3
+Requires-Dist: databricks-sdk==0.52.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
 
 # Zipher SDK
 
@@ -120,3 +143,70 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 # Continue with sending the optimized configuration to Databricks via the Databricks python SDK, Airflow operator, etc.
 
 ```
+
+### Update Existing Multiple Tasks Configuration
+
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
+
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```

{zipher-sdk-0.3.0 → zipher_sdk-0.4.1}/zipher_sdk.egg-info/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 zipher = zipher.cli:main
-

zipher-sdk-0.3.0/zipher/__init__.py

@@ -1 +0,0 @@
-from .client import Client