databricks-sdk 0.43.0__py3-none-any.whl → 0.44.1__py3-none-any.whl

databricks/sdk/credentials_provider.py

@@ -9,6 +9,7 @@ import pathlib
 import platform
 import subprocess
 import sys
+import threading
 import time
 from datetime import datetime
 from typing import Callable, Dict, List, Optional, Tuple, Union
@@ -723,14 +724,17 @@ def metadata_service(cfg: 'Config') -> Optional[CredentialsProvider]:
 # This Code is derived from Mlflow DatabricksModelServingConfigProvider
 # https://github.com/mlflow/mlflow/blob/1219e3ef1aac7d337a618a352cd859b336cf5c81/mlflow/legacy_databricks_cli/configure/provider.py#L332
 class ModelServingAuthProvider():
+    USER_CREDENTIALS = "user_credentials"
+
     _MODEL_DEPENDENCY_OAUTH_TOKEN_FILE_PATH = "/var/credentials-secret/model-dependencies-oauth-token"
 
-    def __init__(self):
+    def __init__(self, credential_type: Optional[str]):
         self.expiry_time = -1
         self.current_token = None
         self.refresh_duration = 300 # 300 Seconds
+        self.credential_type = credential_type
 
-    def should_fetch_model_serving_environment_oauth(self) -> bool:
+    def should_fetch_model_serving_environment_oauth() -> bool:
         """
         Check whether this is the model serving environment
         Additionally check if the oauth token file path exists
@@ -739,15 +743,15 @@ class ModelServingAuthProvider():
         is_in_model_serving_env = (os.environ.get("IS_IN_DB_MODEL_SERVING_ENV")
                                    or os.environ.get("IS_IN_DATABRICKS_MODEL_SERVING_ENV") or "false")
         return (is_in_model_serving_env == "true"
-                and os.path.isfile(self._MODEL_DEPENDENCY_OAUTH_TOKEN_FILE_PATH))
+                and os.path.isfile(ModelServingAuthProvider._MODEL_DEPENDENCY_OAUTH_TOKEN_FILE_PATH))
 
-    def get_model_dependency_oauth_token(self, should_retry=True) -> str:
+    def _get_model_dependency_oauth_token(self, should_retry=True) -> str:
         # Use Cached value if it is valid
         if self.current_token is not None and self.expiry_time > time.time():
             return self.current_token
 
         try:
-            with open(self._MODEL_DEPENDENCY_OAUTH_TOKEN_FILE_PATH) as f:
+            with open(ModelServingAuthProvider._MODEL_DEPENDENCY_OAUTH_TOKEN_FILE_PATH) as f:
                 oauth_dict = json.load(f)
                 self.current_token = oauth_dict["OAUTH_TOKEN"][0]["oauthTokenValue"]
                 self.expiry_time = time.time() + self.refresh_duration
@@ -757,32 +761,43 @@ class ModelServingAuthProvider():
                 logger.warning("Unable to read oauth token on first attmept in Model Serving Environment",
                                exc_info=e)
                 time.sleep(0.5)
-                return self.get_model_dependency_oauth_token(should_retry=False)
+                return self._get_model_dependency_oauth_token(should_retry=False)
             else:
                 raise RuntimeError(
                     "Unable to read OAuth credentials from the file mounted in Databricks Model Serving"
                 ) from e
         return self.current_token
 
+    def _get_invokers_token(self):
+        current_thread = threading.current_thread()
+        thread_data = current_thread.__dict__
+        invokers_token = None
+        if "invokers_token" in thread_data:
+            invokers_token = thread_data["invokers_token"]
+
+        if invokers_token is None:
+            raise RuntimeError("Unable to read Invokers Token in Databricks Model Serving")
+
+        return invokers_token
+
     def get_databricks_host_token(self) -> Optional[Tuple[str, str]]:
-        if not self.should_fetch_model_serving_environment_oauth():
+        if not ModelServingAuthProvider.should_fetch_model_serving_environment_oauth():
             return None
 
         # read from DB_MODEL_SERVING_HOST_ENV_VAR if available otherwise MODEL_SERVING_HOST_ENV_VAR
         host = os.environ.get("DATABRICKS_MODEL_SERVING_HOST_URL") or os.environ.get(
             "DB_MODEL_SERVING_HOST_URL")
-        token = self.get_model_dependency_oauth_token()
 
-        return (host, token)
+        if self.credential_type == ModelServingAuthProvider.USER_CREDENTIALS:
+            return (host, self._get_invokers_token())
+        else:
+            return (host, self._get_model_dependency_oauth_token())
 
 
-@credentials_strategy('model-serving', [])
-def model_serving_auth(cfg: 'Config') -> Optional[CredentialsProvider]:
+def model_serving_auth_visitor(cfg: 'Config',
+                               credential_type: Optional[str] = None) -> Optional[CredentialsProvider]:
     try:
-        model_serving_auth_provider = ModelServingAuthProvider()
-        if not model_serving_auth_provider.should_fetch_model_serving_environment_oauth():
-            logger.debug("model-serving: Not in Databricks Model Serving, skipping")
-            return None
+        model_serving_auth_provider = ModelServingAuthProvider(credential_type)
         host, token = model_serving_auth_provider.get_databricks_host_token()
         if token is None:
             raise ValueError(
@@ -793,7 +808,6 @@ def model_serving_auth(cfg: 'Config') -> Optional[CredentialsProvider]:
     except Exception as e:
         logger.warning("Unable to get auth from Databricks Model Serving Environment", exc_info=e)
         return None
-
     logger.info("Using Databricks Model Serving Authentication")
 
     def inner() -> Dict[str, str]:
@@ -804,6 +818,15 @@ def model_serving_auth(cfg: 'Config') -> Optional[CredentialsProvider]:
     return inner
 
 
+@credentials_strategy('model-serving', [])
+def model_serving_auth(cfg: 'Config') -> Optional[CredentialsProvider]:
+    if not ModelServingAuthProvider.should_fetch_model_serving_environment_oauth():
+        logger.debug("model-serving: Not in Databricks Model Serving, skipping")
+        return None
+
+    return model_serving_auth_visitor(cfg)
+
+
 class DefaultCredentials:
     """ Select the first applicable credential provider from the chain """
 
@@ -846,3 +869,35 @@ class DefaultCredentials:
         raise ValueError(
             f'cannot configure default credentials, please check {auth_flow_url} to configure credentials for your preferred authentication method.'
         )
+
+
+class ModelServingUserCredentials(CredentialsStrategy):
+    """
+    This credential strategy is designed for authenticating the Databricks SDK in the model serving environment using user-specific rights. 
+    In the model serving environment, the strategy retrieves a downscoped user token from the thread-local variable. 
+    In any other environments, the class defaults to the DefaultCredentialStrategy. 
+    To use this credential strategy, instantiate the WorkspaceClient with the ModelServingUserCredentials strategy as follows:
+
+    invokers_client = WorkspaceClient(credential_strategy = ModelServingUserCredentials())
+    """
+
+    def __init__(self):
+        self.credential_type = ModelServingAuthProvider.USER_CREDENTIALS
+        self.default_credentials = DefaultCredentials()
+
+    def auth_type(self):
+        if ModelServingAuthProvider.should_fetch_model_serving_environment_oauth():
+            return "model_serving_" + self.credential_type
+        else:
+            return self.default_credentials.auth_type()
+
+    def __call__(self, cfg: 'Config') -> CredentialsProvider:
+        if ModelServingAuthProvider.should_fetch_model_serving_environment_oauth():
+            header_factory = model_serving_auth_visitor(cfg, self.credential_type)
+            if not header_factory:
+                raise ValueError(
+                    f"Unable to authenticate using {self.credential_type} in Databricks Model Serving Environment"
+                )
+            return header_factory
+        else:
+            return self.default_credentials(cfg)

databricks/sdk/mixins/jobs.py

@@ -11,9 +11,10 @@ class JobsExt(jobs.JobsAPI):
                 include_history: Optional[bool] = None,
                 include_resolved_values: Optional[bool] = None,
                 page_token: Optional[str] = None) -> jobs.Run:
-        """
-        This method fetches the details of a run identified by `run_id`. If the run has multiple pages of tasks or iterations,
-        it will paginate through all pages and aggregate the results.
+        """Get a single job run.
+
+        Retrieve the metadata of a run. If a run has multiple pages of tasks, it will paginate through all pages of tasks, iterations, job_clusters, job_parameters, and repair history.
+
         :param run_id: int
           The canonical identifier of the run for which to retrieve the metadata. This field is required.
         :param include_history: bool (optional)
@@ -21,8 +22,9 @@ class JobsExt(jobs.JobsAPI):
         :param include_resolved_values: bool (optional)
           Whether to include resolved parameter values in the response.
         :param page_token: str (optional)
-          To list the next page or the previous page of job tasks, set this field to the value of the
-          `next_page_token` or `prev_page_token` returned in the GetJob response.
+          To list the next page of job tasks, set this field to the value of the `next_page_token` returned in
+          the GetJob response.
+
         :returns: :class:`Run`
         """
         run = super().get_run(run_id,
@@ -34,6 +36,7 @@ class JobsExt(jobs.JobsAPI):
         # When querying a ForEach task run, a page token is returned when there are more than 100 iterations. Only a single task is returned, corresponding to the ForEach task itself. Therefore, the client only reads the iterations from the next page and not the tasks.
         is_paginating_iterations = run.iterations is not None and len(run.iterations) > 0
 
+        # runs/get response includes next_page_token as long as there are more pages to fetch.
         while run.next_page_token is not None:
             next_run = super().get_run(run_id,
                                        include_history=include_history,
@@ -43,7 +46,10 @@ class JobsExt(jobs.JobsAPI):
                 run.iterations.extend(next_run.iterations)
             else:
                 run.tasks.extend(next_run.tasks)
+            # Each new page of runs/get response includes the next page of the job_clusters, job_parameters, and repair history.
+            run.job_clusters.extend(next_run.job_clusters)
+            run.job_parameters.extend(next_run.job_parameters)
+            run.repair_history.extend(next_run.repair_history)
             run.next_page_token = next_run.next_page_token
 
-        run.prev_page_token = None
         return run

databricks/sdk/service/apps.py

@@ -45,6 +45,9 @@ class App:
     description: Optional[str] = None
     """The description of the app."""
 
+    id: Optional[str] = None
+    """The unique identifier of the app."""
+
     pending_deployment: Optional[AppDeployment] = None
     """The pending deployment of the app. A deployment is considered pending when it is being prepared
     for deployment to the app compute."""
@@ -78,6 +81,7 @@ class App:
         if self.default_source_code_path is not None:
             body['default_source_code_path'] = self.default_source_code_path
         if self.description is not None: body['description'] = self.description
+        if self.id is not None: body['id'] = self.id
         if self.name is not None: body['name'] = self.name
         if self.pending_deployment: body['pending_deployment'] = self.pending_deployment.as_dict()
         if self.resources: body['resources'] = [v.as_dict() for v in self.resources]
@@ -102,6 +106,7 @@ class App:
         if self.default_source_code_path is not None:
             body['default_source_code_path'] = self.default_source_code_path
         if self.description is not None: body['description'] = self.description
+        if self.id is not None: body['id'] = self.id
         if self.name is not None: body['name'] = self.name
         if self.pending_deployment: body['pending_deployment'] = self.pending_deployment
         if self.resources: body['resources'] = self.resources
@@ -125,6 +130,7 @@ class App:
                    creator=d.get('creator', None),
                    default_source_code_path=d.get('default_source_code_path', None),
                    description=d.get('description', None),
+                   id=d.get('id', None),
                    name=d.get('name', None),
                    pending_deployment=_from_dict(d, 'pending_deployment', AppDeployment),
                    resources=_repeated_dict(d, 'resources', AppResource),

databricks/sdk/service/billing.py

@@ -894,6 +894,27 @@ class GetBudgetConfigurationResponse:
         return cls(budget=_from_dict(d, 'budget', BudgetConfiguration))
 
 
+@dataclass
+class LimitConfig:
+    """The limit configuration of the policy. Limit configuration provide a budget policy level cost
+    control by enforcing the limit."""
+
+    def as_dict(self) -> dict:
+        """Serializes the LimitConfig into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the LimitConfig into a shallow dictionary of its immediate attributes."""
+        body = {}
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> LimitConfig:
+        """Deserializes the LimitConfig from a dictionary."""
+        return cls()
+
+
 @dataclass
 class ListBudgetConfigurationsResponse:
     budgets: Optional[List[BudgetConfiguration]] = None
@@ -1641,23 +1662,32 @@ class BudgetPolicyAPI:
                 return
             query['page_token'] = json['next_page_token']
 
-    def update(self, policy_id: str, *, policy: Optional[BudgetPolicy] = None) -> BudgetPolicy:
+    def update(self,
+               policy_id: str,
+               *,
+               limit_config: Optional[LimitConfig] = None,
+               policy: Optional[BudgetPolicy] = None) -> BudgetPolicy:
         """Update a budget policy.
         
         Updates a policy
         
         :param policy_id: str
           The Id of the policy. This field is generated by Databricks and globally unique.
+        :param limit_config: :class:`LimitConfig` (optional)
+          DEPRECATED. This is redundant field as LimitConfig is part of the BudgetPolicy
         :param policy: :class:`BudgetPolicy` (optional)
           Contains the BudgetPolicy details.
         
         :returns: :class:`BudgetPolicy`
         """
         body = policy.as_dict()
+        query = {}
+        if limit_config is not None: query['limit_config'] = limit_config.as_dict()
         headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
 
         res = self._api.do('PATCH',
                            f'/api/2.1/accounts/{self._api.account_id}/budget-policies/{policy_id}',
+                           query=query,
                            body=body,
                            headers=headers)
         return BudgetPolicy.from_dict(res)

databricks/sdk/service/catalog.py

@@ -8983,6 +8983,7 @@ class CatalogsAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET', '/api/2.1/unity-catalog/catalogs', query=query, headers=headers)
             if 'catalogs' in json:
@@ -9151,6 +9152,7 @@ class ConnectionsAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET', '/api/2.1/unity-catalog/connections', query=query, headers=headers)
             if 'connections' in json:
@@ -9656,6 +9658,7 @@ class ExternalLocationsAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET',
                                 '/api/2.1/unity-catalog/external-locations',
@@ -11389,6 +11392,7 @@ class SchemasAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET', '/api/2.1/unity-catalog/schemas', query=query, headers=headers)
             if 'schemas' in json:
@@ -11578,6 +11582,7 @@ class StorageCredentialsAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET',
                                 '/api/2.1/unity-catalog/storage-credentials',
@@ -11802,6 +11807,7 @@ class SystemSchemasAPI:
         if page_token is not None: query['page_token'] = page_token
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET',
                                 f'/api/2.1/unity-catalog/metastores/{metastore_id}/systemschemas',
@@ -12044,6 +12050,7 @@ class TablesAPI:
         if schema_name is not None: query['schema_name'] = schema_name
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET', '/api/2.1/unity-catalog/tables', query=query, headers=headers)
             if 'tables' in json:
@@ -12104,6 +12111,7 @@ class TablesAPI:
         if table_name_pattern is not None: query['table_name_pattern'] = table_name_pattern
         headers = {'Accept': 'application/json', }
 
+        if "max_results" not in query: query['max_results'] = 0
         while True:
             json = self._api.do('GET', '/api/2.1/unity-catalog/table-summaries', query=query, headers=headers)
             if 'tables' in json:

databricks/sdk/service/cleanrooms.py

@@ -289,24 +289,11 @@ class CleanRoomAssetNotebook:
     """Base 64 representation of the notebook contents. This is the same format as returned by
     :method:workspace/export with the format of **HTML**."""
 
-    review_state: Optional[CleanRoomNotebookReviewNotebookReviewState] = None
-    """top-level status derived from all reviews"""
-
-    reviews: Optional[List[CleanRoomNotebookReview]] = None
-    """All existing approvals or rejections"""
-
-    runner_collaborators: Optional[List[CleanRoomCollaborator]] = None
-    """collaborators that can run the notebook"""
-
     def as_dict(self) -> dict:
         """Serializes the CleanRoomAssetNotebook into a dictionary suitable for use as a JSON request body."""
         body = {}
         if self.etag is not None: body['etag'] = self.etag
         if self.notebook_content is not None: body['notebook_content'] = self.notebook_content
-        if self.review_state is not None: body['review_state'] = self.review_state.value
-        if self.reviews: body['reviews'] = [v.as_dict() for v in self.reviews]
-        if self.runner_collaborators:
-            body['runner_collaborators'] = [v.as_dict() for v in self.runner_collaborators]
         return body
 
     def as_shallow_dict(self) -> dict:
@@ -314,19 +301,12 @@ class CleanRoomAssetNotebook:
         body = {}
         if self.etag is not None: body['etag'] = self.etag
         if self.notebook_content is not None: body['notebook_content'] = self.notebook_content
-        if self.review_state is not None: body['review_state'] = self.review_state
-        if self.reviews: body['reviews'] = self.reviews
-        if self.runner_collaborators: body['runner_collaborators'] = self.runner_collaborators
         return body
 
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> CleanRoomAssetNotebook:
         """Deserializes the CleanRoomAssetNotebook from a dictionary."""
-        return cls(etag=d.get('etag', None),
-                   notebook_content=d.get('notebook_content', None),
-                   review_state=_enum(d, 'review_state', CleanRoomNotebookReviewNotebookReviewState),
-                   reviews=_repeated_dict(d, 'reviews', CleanRoomNotebookReview),
-                   runner_collaborators=_repeated_dict(d, 'runner_collaborators', CleanRoomCollaborator))
+        return cls(etag=d.get('etag', None), notebook_content=d.get('notebook_content', None))
 
 
 class CleanRoomAssetStatusEnum(Enum):
@@ -531,56 +511,6 @@ class CleanRoomCollaborator:
                    organization_name=d.get('organization_name', None))
 
 
-@dataclass
-class CleanRoomNotebookReview:
-    comment: Optional[str] = None
-    """review comment"""
-
-    created_at_millis: Optional[int] = None
-    """timestamp of when the review was submitted"""
-
-    review_state: Optional[CleanRoomNotebookReviewNotebookReviewState] = None
-    """review outcome"""
-
-    reviewer_collaborator_alias: Optional[str] = None
-    """collaborator alias of the reviewer"""
-
-    def as_dict(self) -> dict:
-        """Serializes the CleanRoomNotebookReview into a dictionary suitable for use as a JSON request body."""
-        body = {}
-        if self.comment is not None: body['comment'] = self.comment
-        if self.created_at_millis is not None: body['created_at_millis'] = self.created_at_millis
-        if self.review_state is not None: body['review_state'] = self.review_state.value
-        if self.reviewer_collaborator_alias is not None:
-            body['reviewer_collaborator_alias'] = self.reviewer_collaborator_alias
-        return body
-
-    def as_shallow_dict(self) -> dict:
-        """Serializes the CleanRoomNotebookReview into a shallow dictionary of its immediate attributes."""
-        body = {}
-        if self.comment is not None: body['comment'] = self.comment
-        if self.created_at_millis is not None: body['created_at_millis'] = self.created_at_millis
-        if self.review_state is not None: body['review_state'] = self.review_state
-        if self.reviewer_collaborator_alias is not None:
-            body['reviewer_collaborator_alias'] = self.reviewer_collaborator_alias
-        return body
-
-    @classmethod
-    def from_dict(cls, d: Dict[str, any]) -> CleanRoomNotebookReview:
-        """Deserializes the CleanRoomNotebookReview from a dictionary."""
-        return cls(comment=d.get('comment', None),
-                   created_at_millis=d.get('created_at_millis', None),
-                   review_state=_enum(d, 'review_state', CleanRoomNotebookReviewNotebookReviewState),
-                   reviewer_collaborator_alias=d.get('reviewer_collaborator_alias', None))
-
-
-class CleanRoomNotebookReviewNotebookReviewState(Enum):
-
-    APPROVED = 'APPROVED'
-    PENDING = 'PENDING'
-    REJECTED = 'REJECTED'
-
-
 @dataclass
 class CleanRoomNotebookTaskRun:
     """Stores information about a single task run."""

databricks/sdk/service/compute.py

@@ -637,11 +637,11 @@ class ClusterAttributes:
     a set of default values will be used."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_name: Optional[str] = None
     """Cluster name requested by the user. This doesn't have to be unique. If not specified at
@@ -947,11 +947,11 @@ class ClusterDetails:
     while each new cluster has a globally unique id."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_log_status: Optional[LogSyncStatus] = None
     """Cluster log delivery status."""
@@ -1428,11 +1428,16 @@ class ClusterLogConf:
     access s3, please make sure the cluster iam role in `instance_profile_arn` has permission to
     write data to the s3 destination."""
 
+    volumes: Optional[VolumesStorageInfo] = None
+    """destination needs to be provided. e.g. `{ "volumes" : { "destination" :
+    "/Volumes/catalog/schema/volume/cluster_log" } }`"""
+
     def as_dict(self) -> dict:
         """Serializes the ClusterLogConf into a dictionary suitable for use as a JSON request body."""
         body = {}
         if self.dbfs: body['dbfs'] = self.dbfs.as_dict()
         if self.s3: body['s3'] = self.s3.as_dict()
+        if self.volumes: body['volumes'] = self.volumes.as_dict()
         return body
 
     def as_shallow_dict(self) -> dict:
@@ -1440,12 +1445,15 @@ class ClusterLogConf:
         body = {}
         if self.dbfs: body['dbfs'] = self.dbfs
         if self.s3: body['s3'] = self.s3
+        if self.volumes: body['volumes'] = self.volumes
         return body
 
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> ClusterLogConf:
         """Deserializes the ClusterLogConf from a dictionary."""
-        return cls(dbfs=_from_dict(d, 'dbfs', DbfsStorageInfo), s3=_from_dict(d, 's3', S3StorageInfo))
+        return cls(dbfs=_from_dict(d, 'dbfs', DbfsStorageInfo),
+                   s3=_from_dict(d, 's3', S3StorageInfo),
+                   volumes=_from_dict(d, 'volumes', VolumesStorageInfo))
 
 
 @dataclass
@@ -1918,11 +1926,11 @@ class ClusterSpec:
     a set of default values will be used."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_name: Optional[str] = None
     """Cluster name requested by the user. This doesn't have to be unique. If not specified at
@@ -2334,11 +2342,11 @@ class CreateCluster:
     cluster."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_name: Optional[str] = None
     """Cluster name requested by the user. This doesn't have to be unique. If not specified at
@@ -3469,11 +3477,11 @@ class EditCluster:
     a set of default values will be used."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_name: Optional[str] = None
     """Cluster name requested by the user. This doesn't have to be unique. If not specified at
@@ -7773,11 +7781,11 @@ class UpdateClusterResource:
     a set of default values will be used."""
 
     cluster_log_conf: Optional[ClusterLogConf] = None
-    """The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-    destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster.
-    If the conf is given, the logs will be delivered to the destination every `5 mins`. The
-    destination of driver logs is `$destination/$clusterId/driver`, while the destination of
-    executor logs is `$destination/$clusterId/executor`."""
+    """The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+    destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+    specified for one cluster. If the conf is given, the logs will be delivered to the destination
+    every `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the
+    destination of executor logs is `$destination/$clusterId/executor`."""
 
     cluster_name: Optional[str] = None
     """Cluster name requested by the user. This doesn't have to be unique. If not specified at
@@ -8077,7 +8085,7 @@ class UpdateResponse:
 @dataclass
 class VolumesStorageInfo:
     destination: str
-    """Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`"""
+    """Unity Catalog volumes file destination, e.g. `/Volumes/catalog/schema/volume/dir/file`"""
 
     def as_dict(self) -> dict:
         """Serializes the VolumesStorageInfo into a dictionary suitable for use as a JSON request body."""
@@ -8619,11 +8627,11 @@ class ClustersAPI:
         :param clone_from: :class:`CloneCluster` (optional)
           When specified, this clones libraries from a source cluster during the creation of a new cluster.
         :param cluster_log_conf: :class:`ClusterLogConf` (optional)
-          The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-          destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If
-          the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of
-          driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is
-          `$destination/$clusterId/executor`.
+          The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+          destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+          specified for one cluster. If the conf is given, the logs will be delivered to the destination every
+          `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination
+          of executor logs is `$destination/$clusterId/executor`.
         :param cluster_name: str (optional)
           Cluster name requested by the user. This doesn't have to be unique. If not specified at creation,
           the cluster name will be an empty string.
@@ -8952,11 +8960,11 @@ class ClustersAPI:
           Attributes related to clusters running on Microsoft Azure. If not specified at cluster creation, a
           set of default values will be used.
         :param cluster_log_conf: :class:`ClusterLogConf` (optional)
-          The configuration for delivering spark logs to a long-term storage destination. Two kinds of
-          destinations (dbfs and s3) are supported. Only one destination can be specified for one cluster. If
-          the conf is given, the logs will be delivered to the destination every `5 mins`. The destination of
-          driver logs is `$destination/$clusterId/driver`, while the destination of executor logs is
-          `$destination/$clusterId/executor`.
+          The configuration for delivering spark logs to a long-term storage destination. Three kinds of
+          destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be
+          specified for one cluster. If the conf is given, the logs will be delivered to the destination every
+          `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while the destination
+          of executor logs is `$destination/$clusterId/executor`.
         :param cluster_name: str (optional)
           Cluster name requested by the user. This doesn't have to be unique. If not specified at creation,
           the cluster name will be an empty string.