zipher-sdk 0.3.0__tar.gz → 0.3.9__tar.gz

{zipher-sdk-0.3.0/zipher_sdk.egg-info → zipher-sdk-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zipher-sdk
-Version: 0.3.0
+Version: 0.3.9
 Summary: Python SDK for programmatic access to the Zipher API
 Home-page: UNKNOWN
 Author: Zipher Inc.
@@ -136,4 +136,70 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 
 ```
 
+### Update Existing Multiple Tasks Configuration
+
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
+
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```
 

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/README.md

@@ -120,3 +120,70 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 # Continue with sending the optimized configuration to Databricks via the Databricks python SDK, Airflow operator, etc.
 
 ```
+
+### Update Existing Multiple Tasks Configuration
+
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
+
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/setup.py

@@ -8,7 +8,7 @@ with open('README.md', encoding='utf-8') as f:
 
 setup(
     name='zipher-sdk',
-    version='0.3.0',
+    version='0.3.9',
     description='Python SDK for programmatic access to the Zipher API',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -16,7 +16,7 @@ setup(
     author_email='yoav@zipher.cloud',
     packages=find_packages(),
     install_requires=[
-        "pydantic==2.11.3",
+        "pydantic>=2.7.3,<3.0.0",
         "requests==2.32.3",
         "databricks-sdk==0.52.0"
     ],

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/tests/test_client.py

@@ -49,6 +49,9 @@ class TestClient(unittest.TestCase):
         res = self.client.update_existing_conf(job_id="1234", existing_conf=config_to_update)
         self.assertDictEqual(config_to_update, res)
 
+    def test_pydantic_version(self):
+        import pydantic
+        print(pydantic.__version__)
 
 if __name__ == '__main__':
     unittest.main()

zipher-sdk-0.3.9/zipher/__init__.py

@@ -0,0 +1,2 @@
+from .client import Client
+from .models import Config

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/zipher/acl/zipher_installer.py

@@ -122,7 +122,7 @@ class ZipherInstaller:
         start_date = datetime.now() - timedelta(days=days_back)
         start_date_ms_int = int(start_date.timestamp() * 1000)
 
-        jobs = self.dbx_client.jobs.list(limit=100)
+        jobs = self.dbx_client.jobs.list()
         runs = self.dbx_client.jobs.list_runs(limit=0, start_time_from=start_date_ms_int)
 
         all_job_ids = set()

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/zipher/client.py

@@ -4,11 +4,9 @@ from typing import Dict, Optional
 
 import requests
 from databricks.sdk.service.compute import ClusterSpec
-
-from zipher.models import ConfFetcherRequest, Config
+from zipher.models import ConfFetcherRequest, GetOptimizedTasksRequest, Config
 from zipher.exceptions import MissingAPIKeyError
 
-
 class Client:
     def __init__(self, customer_id: str, zipher_api_key: str = '', config: Optional[Config] = None):
         self.config = config or Config()
@@ -33,17 +31,35 @@ class Client:
         else:
             response.raise_for_status()
 
-    def get_optimized_config(self, job_id: str) -> Dict:
-        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id)
+    def _call_get_optimized_tasks_api(self, params: GetOptimizedTasksRequest):
+        headers = {
+            'x-api-key': self.zipher_api_key
+        }
+        response = requests.get(
+            url=self.config.zipher_get_optimized_tasks_api_endpoint,
+            headers=headers,
+            params=params.model_dump()
+        )
+        if response.ok:
+            return response.json()
+        else:
+            response.raise_for_status()
+
+    def get_optimized_tasks(self, job_id: str, tasks: list[dict]) -> list[dict]:
+        params = GetOptimizedTasksRequest(customer_id=self.customer_id, job_id=job_id, tasks=json.dumps(tasks))
+        return self._call_get_optimized_tasks_api(params)
+    
+    def get_optimized_config(self, job_id: str, task_key: Optional[str] = None) -> Dict:
+        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, task_key=task_key)
         return self._call_conf_fetcher_api(params)
 
     def get_optimized_config_as_dbx_cluster_spec(self, job_id: str) -> ClusterSpec:
         return ClusterSpec.from_dict(self.get_optimized_config(job_id))
 
-    def update_existing_conf(self, job_id: str, existing_conf: Dict) -> Dict:
+    def update_existing_conf(self, job_id: str, existing_conf: Dict, task_key: Optional[str] = None) -> Dict:
         if 'new_cluster' in existing_conf:
             existing_conf = existing_conf['new_cluster']
-        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, merge_with=json.dumps(existing_conf))
+        params = ConfFetcherRequest(customer_id=self.customer_id, job_id=job_id, merge_with=json.dumps(existing_conf), task_key=task_key)
         try:
             return self._call_conf_fetcher_api(params)
         except:

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/zipher/models.py

@@ -12,6 +12,7 @@ class WorkspaceClientAuthType(Enum):
 class ConfFetcherRequest(BaseModel):
     job_id: str
     customer_id: str
+    task_key: Optional[str] = None
     merge_with: Optional[str] = None
 
 
@@ -24,3 +25,9 @@ class Config(BaseModel, extra='forbid', use_enum_values=True):
     databricks_client_secret_env_var: str = 'DATABRICKS_CLIENT_SECRET'
 
     zipher_config_fetcher_api_endpoint: str = 'https://api.zipher.cloud/default/get_clusterspec'
+    zipher_get_optimized_tasks_api_endpoint: str = 'https://api.zipher.cloud/default/get_optimized_tasks'
+
+class GetOptimizedTasksRequest(BaseModel):
+    customer_id: str
+    job_id: str
+    tasks: Optional[str] = None

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9/zipher_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: zipher-sdk
-Version: 0.3.0
+Version: 0.3.9
 Summary: Python SDK for programmatic access to the Zipher API
 Home-page: UNKNOWN
 Author: Zipher Inc.
@@ -136,4 +136,70 @@ optimized_cluster = client.update_existing_conf(job_id="my-job-id", existing_con
 
 ```
 
+### Update Existing Multiple Tasks Configuration
+
+You can update multiple databricks tasks by initializing a zipher Client and sending a JSON representing a list of dbx SubmitTask objects to
+the `get_optimized_tasks` function. 
+
+```python
+from zipher import Client
+
+client = Client(customer_id="my_customer_id")  # assuming the zipher API key is stored in ZIPHER_API_KEY environment variable
+
+tasks_to_optimize = [
+    {
+        "task_key": "task_1",
+        "description": "Test notebook task",
+        "notebook_task": {
+            "notebook_path": "/path/to/your/notebook",
+            "base_parameters": {
+                "param1": "value1"
+            }
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "aws_attributes": {
+                "first_on_demand": 0,
+                "availability": "SPOT",
+                "zone_id": "auto",
+                "spot_bid_price_percent": 100,
+                "ebs_volume_count": 0
+            },
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        }
+    },
+    {
+        "task_key": "task_2",
+        "description": "Test Python task",
+        "spark_python_task": {
+            "python_file": "/path/to/your/python_file.py",
+        },
+        "new_cluster": {
+            "spark_version": "14.3.x-scala2.12",
+            "node_type_id": "m6id.large",
+            "driver_node_type_id": "m6id.large",
+            "num_workers": 2,
+            "spark_conf": {
+                "spark.driver.maxResultSize": "4g"
+            }
+        },
+        "timeout_seconds": 3600,
+        "depends_on": [
+            {
+                "task_key": "task_1"
+            }
+        ]
+    }
+]
+
+# Update tasks
+optimized_tasks = client.get_optimized_tasks(job_id="my-job-id", tasks=tasks_to_optimize)
+
+# Continue with sending the optimized tasks to Databricks via the Databricks python SDK, Airflow operator, etc.
+```
 

{zipher-sdk-0.3.0 → zipher-sdk-0.3.9}/zipher_sdk.egg-info/requires.txt

@@ -1,3 +1,3 @@
-pydantic==2.11.3
+pydantic<3.0.0,>=2.7.3
 requests==2.32.3
 databricks-sdk==0.52.0

zipher-sdk-0.3.0/zipher/__init__.py

@@ -1 +0,0 @@
-from .client import Client