databricks-sdk 0.65.0__py3-none-any.whl → 0.67.0__py3-none-any.whl

databricks/sdk/service/jobs.py

@@ -1651,9 +1651,7 @@ class ExportRunOutput:
 
     views: Optional[List[ViewItem]] = None
     """The exported content in HTML format (one for every view item). To extract the HTML notebook from
-    the JSON response, download and run this [Python script].
-    
-    [Python script]: https://docs.databricks.com/en/_static/examples/extract.py"""
+    the JSON response, download and run this [Python script](/_static/examples/extract.py)."""
 
     def as_dict(self) -> dict:
         """Serializes the ExportRunOutput into a dictionary suitable for use as a JSON request body."""
@@ -5659,7 +5657,7 @@ class RunTask:
     clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None
     """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present.
     
-    [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html"""
+    [clean rooms]: https://docs.databricks.com/clean-rooms/index.html"""
 
     cleanup_duration: Optional[int] = None
     """The time in milliseconds it took to terminate the cluster and clean up any associated artifacts.
@@ -5696,9 +5694,6 @@ class RunTask:
     description: Optional[str] = None
     """An optional description for this task."""
 
-    disabled: Optional[bool] = None
-    """Deprecated, field was never used in production."""
-
     effective_performance_target: Optional[PerformanceTarget] = None
     """The actual performance target used by the serverless run during execution. This can differ from
     the client-set performance target on the request depending on whether the performance mode is
@@ -5810,21 +5805,9 @@ class RunTask:
     """The task runs a Python file when the `spark_python_task` field is present."""
 
     spark_submit_task: Optional[SparkSubmitTask] = None
-    """(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present.
-    This task can run only on new clusters and is not compatible with serverless compute.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use
-    `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark
-    configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you
-    _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for
-    Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value
-    to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths."""
+    """(Legacy) The task runs the spark-submit script when the spark_submit_task field is present.
+    Databricks recommends using the spark_jar_task instead; see [Spark Submit task for
+    jobs](/jobs/spark-submit)."""
 
     sql_task: Optional[SqlTask] = None
     """The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when
@@ -5873,8 +5856,6 @@ class RunTask:
             body["depends_on"] = [v.as_dict() for v in self.depends_on]
         if self.description is not None:
             body["description"] = self.description
-        if self.disabled is not None:
-            body["disabled"] = self.disabled
         if self.effective_performance_target is not None:
             body["effective_performance_target"] = self.effective_performance_target.value
         if self.email_notifications:
@@ -5972,8 +5953,6 @@ class RunTask:
             body["depends_on"] = self.depends_on
         if self.description is not None:
             body["description"] = self.description
-        if self.disabled is not None:
-            body["disabled"] = self.disabled
         if self.effective_performance_target is not None:
             body["effective_performance_target"] = self.effective_performance_target
         if self.email_notifications:
@@ -6061,7 +6040,6 @@ class RunTask:
             dbt_task=_from_dict(d, "dbt_task", DbtTask),
             depends_on=_repeated_dict(d, "depends_on", TaskDependency),
             description=d.get("description", None),
-            disabled=d.get("disabled", None),
             effective_performance_target=_enum(d, "effective_performance_target", PerformanceTarget),
             email_notifications=_from_dict(d, "email_notifications", JobEmailNotifications),
             end_time=d.get("end_time", None),
@@ -6916,7 +6894,7 @@ class SubmitTask:
     clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None
     """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present.
     
-    [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html"""
+    [clean rooms]: https://docs.databricks.com/clean-rooms/index.html"""
 
     condition_task: Optional[ConditionTask] = None
     """The task evaluates a condition that can be used to control the execution of other tasks when the
@@ -7003,21 +6981,9 @@ class SubmitTask:
     """The task runs a Python file when the `spark_python_task` field is present."""
 
     spark_submit_task: Optional[SparkSubmitTask] = None
-    """(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present.
-    This task can run only on new clusters and is not compatible with serverless compute.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use
-    `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark
-    configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you
-    _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for
-    Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value
-    to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths."""
+    """(Legacy) The task runs the spark-submit script when the spark_submit_task field is present.
+    Databricks recommends using the spark_jar_task instead; see [Spark Submit task for
+    jobs](/jobs/spark-submit)."""
 
     sql_task: Optional[SqlTask] = None
     """The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when
@@ -7407,7 +7373,7 @@ class Task:
     clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None
     """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present.
     
-    [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html"""
+    [clean rooms]: https://docs.databricks.com/clean-rooms/index.html"""
 
     condition_task: Optional[ConditionTask] = None
     """The task evaluates a condition that can be used to control the execution of other tasks when the
@@ -7438,6 +7404,10 @@ class Task:
     disable_auto_optimization: Optional[bool] = None
     """An option to disable auto optimization in serverless"""
 
+    disabled: Optional[bool] = None
+    """An optional flag to disable the task. If set to true, the task will not run even if it is part
+    of a job."""
+
     email_notifications: Optional[TaskEmailNotifications] = None
     """An optional set of email addresses that is notified when runs of this task begin or complete as
     well as when this task is deleted. The default behavior is to not send any emails."""
@@ -7520,21 +7490,9 @@ class Task:
     """The task runs a Python file when the `spark_python_task` field is present."""
 
     spark_submit_task: Optional[SparkSubmitTask] = None
-    """(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present.
-    This task can run only on new clusters and is not compatible with serverless compute.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use
-    `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark
-    configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you
-    _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for
-    Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value
-    to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths."""
+    """(Legacy) The task runs the spark-submit script when the spark_submit_task field is present.
+    Databricks recommends using the spark_jar_task instead; see [Spark Submit task for
+    jobs](/jobs/spark-submit)."""
 
     sql_task: Optional[SqlTask] = None
     """The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when
@@ -7568,6 +7526,8 @@ class Task:
             body["description"] = self.description
         if self.disable_auto_optimization is not None:
             body["disable_auto_optimization"] = self.disable_auto_optimization
+        if self.disabled is not None:
+            body["disabled"] = self.disabled
         if self.email_notifications:
             body["email_notifications"] = self.email_notifications.as_dict()
         if self.environment_key is not None:
@@ -7643,6 +7603,8 @@ class Task:
             body["description"] = self.description
         if self.disable_auto_optimization is not None:
             body["disable_auto_optimization"] = self.disable_auto_optimization
+        if self.disabled is not None:
+            body["disabled"] = self.disabled
         if self.email_notifications:
             body["email_notifications"] = self.email_notifications
         if self.environment_key is not None:
@@ -7710,6 +7672,7 @@ class Task:
             depends_on=_repeated_dict(d, "depends_on", TaskDependency),
             description=d.get("description", None),
             disable_auto_optimization=d.get("disable_auto_optimization", None),
+            disabled=d.get("disabled", None),
             email_notifications=_from_dict(d, "email_notifications", TaskEmailNotifications),
             environment_key=d.get("environment_key", None),
             existing_cluster_id=d.get("existing_cluster_id", None),

databricks/sdk/service/ml.py

@@ -512,6 +512,30 @@ class CreateWebhookResponse:
         return cls(webhook=_from_dict(d, "webhook", RegistryWebhook))
 
 
+@dataclass
+class DataSource:
+    delta_table_source: Optional[DeltaTableSource] = None
+
+    def as_dict(self) -> dict:
+        """Serializes the DataSource into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.delta_table_source:
+            body["delta_table_source"] = self.delta_table_source.as_dict()
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the DataSource into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.delta_table_source:
+            body["delta_table_source"] = self.delta_table_source
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> DataSource:
+        """Deserializes the DataSource from a dictionary."""
+        return cls(delta_table_source=_from_dict(d, "delta_table_source", DeltaTableSource))
+
+
 @dataclass
 class Dataset:
     """Dataset. Represents a reference to data used for training, testing, or evaluation during the
@@ -868,6 +892,49 @@ class DeleteWebhookResponse:
         return cls()
 
 
+@dataclass
+class DeltaTableSource:
+    full_name: str
+    """The full three-part (catalog, schema, table) name of the Delta table."""
+
+    entity_columns: List[str]
+    """The entity columns of the Delta table."""
+
+    timeseries_column: str
+    """The timeseries column of the Delta table."""
+
+    def as_dict(self) -> dict:
+        """Serializes the DeltaTableSource into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.entity_columns:
+            body["entity_columns"] = [v for v in self.entity_columns]
+        if self.full_name is not None:
+            body["full_name"] = self.full_name
+        if self.timeseries_column is not None:
+            body["timeseries_column"] = self.timeseries_column
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the DeltaTableSource into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.entity_columns:
+            body["entity_columns"] = self.entity_columns
+        if self.full_name is not None:
+            body["full_name"] = self.full_name
+        if self.timeseries_column is not None:
+            body["timeseries_column"] = self.timeseries_column
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> DeltaTableSource:
+        """Deserializes the DeltaTableSource from a dictionary."""
+        return cls(
+            entity_columns=d.get("entity_columns", None),
+            full_name=d.get("full_name", None),
+            timeseries_column=d.get("timeseries_column", None),
+        )
+
+
 @dataclass
 class Experiment:
     """An experiment and its metadata."""
@@ -1210,6 +1277,73 @@ class ExperimentTag:
         return cls(key=d.get("key", None), value=d.get("value", None))
 
 
+@dataclass
+class Feature:
+    full_name: str
+    """The full three-part name (catalog, schema, name) of the feature."""
+
+    source: DataSource
+    """The data source of the feature."""
+
+    inputs: List[str]
+    """The input columns from which the feature is computed."""
+
+    function: Function
+    """The function by which the feature is computed."""
+
+    time_window: TimeWindow
+    """The time window in which the feature is computed."""
+
+    description: Optional[str] = None
+    """The description of the feature."""
+
+    def as_dict(self) -> dict:
+        """Serializes the Feature into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.description is not None:
+            body["description"] = self.description
+        if self.full_name is not None:
+            body["full_name"] = self.full_name
+        if self.function:
+            body["function"] = self.function.as_dict()
+        if self.inputs:
+            body["inputs"] = [v for v in self.inputs]
+        if self.source:
+            body["source"] = self.source.as_dict()
+        if self.time_window:
+            body["time_window"] = self.time_window.as_dict()
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the Feature into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.description is not None:
+            body["description"] = self.description
+        if self.full_name is not None:
+            body["full_name"] = self.full_name
+        if self.function:
+            body["function"] = self.function
+        if self.inputs:
+            body["inputs"] = self.inputs
+        if self.source:
+            body["source"] = self.source
+        if self.time_window:
+            body["time_window"] = self.time_window
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> Feature:
+        """Deserializes the Feature from a dictionary."""
+        return cls(
+            description=d.get("description", None),
+            full_name=d.get("full_name", None),
+            function=_from_dict(d, "function", Function),
+            inputs=d.get("inputs", None),
+            source=_from_dict(d, "source", DataSource),
+            time_window=_from_dict(d, "time_window", TimeWindow),
+        )
+
+
 @dataclass
 class FeatureLineage:
     feature_specs: Optional[List[FeatureLineageFeatureSpec]] = None
@@ -1520,6 +1654,90 @@ class ForecastingExperimentState(Enum):
     SUCCEEDED = "SUCCEEDED"
 
 
+@dataclass
+class Function:
+    function_type: FunctionFunctionType
+    """The type of the function."""
+
+    extra_parameters: Optional[List[FunctionExtraParameter]] = None
+    """Extra parameters for parameterized functions."""
+
+    def as_dict(self) -> dict:
+        """Serializes the Function into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.extra_parameters:
+            body["extra_parameters"] = [v.as_dict() for v in self.extra_parameters]
+        if self.function_type is not None:
+            body["function_type"] = self.function_type.value
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the Function into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.extra_parameters:
+            body["extra_parameters"] = self.extra_parameters
+        if self.function_type is not None:
+            body["function_type"] = self.function_type
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> Function:
+        """Deserializes the Function from a dictionary."""
+        return cls(
+            extra_parameters=_repeated_dict(d, "extra_parameters", FunctionExtraParameter),
+            function_type=_enum(d, "function_type", FunctionFunctionType),
+        )
+
+
+@dataclass
+class FunctionExtraParameter:
+    key: str
+    """The name of the parameter."""
+
+    value: str
+    """The value of the parameter."""
+
+    def as_dict(self) -> dict:
+        """Serializes the FunctionExtraParameter into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.key is not None:
+            body["key"] = self.key
+        if self.value is not None:
+            body["value"] = self.value
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the FunctionExtraParameter into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.key is not None:
+            body["key"] = self.key
+        if self.value is not None:
+            body["value"] = self.value
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> FunctionExtraParameter:
+        """Deserializes the FunctionExtraParameter from a dictionary."""
+        return cls(key=d.get("key", None), value=d.get("value", None))
+
+
+class FunctionFunctionType(Enum):
+
+    APPROX_COUNT_DISTINCT = "APPROX_COUNT_DISTINCT"
+    APPROX_PERCENTILE = "APPROX_PERCENTILE"
+    AVG = "AVG"
+    COUNT = "COUNT"
+    FIRST = "FIRST"
+    LAST = "LAST"
+    MAX = "MAX"
+    MIN = "MIN"
+    STDDEV_POP = "STDDEV_POP"
+    STDDEV_SAMP = "STDDEV_SAMP"
+    SUM = "SUM"
+    VAR_POP = "VAR_POP"
+    VAR_SAMP = "VAR_SAMP"
+
+
 @dataclass
 class GetExperimentByNameResponse:
     experiment: Optional[Experiment] = None
@@ -2167,6 +2385,38 @@ class ListFeatureTagsResponse:
         )
 
 
+@dataclass
+class ListFeaturesResponse:
+    features: Optional[List[Feature]] = None
+    """List of features."""
+
+    next_page_token: Optional[str] = None
+    """Pagination token to request the next page of results for this query."""
+
+    def as_dict(self) -> dict:
+        """Serializes the ListFeaturesResponse into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.features:
+            body["features"] = [v.as_dict() for v in self.features]
+        if self.next_page_token is not None:
+            body["next_page_token"] = self.next_page_token
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the ListFeaturesResponse into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.features:
+            body["features"] = self.features
+        if self.next_page_token is not None:
+            body["next_page_token"] = self.next_page_token
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> ListFeaturesResponse:
+        """Deserializes the ListFeaturesResponse from a dictionary."""
+        return cls(features=_repeated_dict(d, "features", Feature), next_page_token=d.get("next_page_token", None))
+
+
 @dataclass
 class ListModelsResponse:
     next_page_token: Optional[str] = None
@@ -4734,6 +4984,38 @@ class TestRegistryWebhookResponse:
         return cls(body=d.get("body", None), status_code=d.get("status_code", None))
 
 
+@dataclass
+class TimeWindow:
+    duration: str
+    """The duration of the time window."""
+
+    offset: Optional[str] = None
+    """The offset of the time window."""
+
+    def as_dict(self) -> dict:
+        """Serializes the TimeWindow into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.duration is not None:
+            body["duration"] = self.duration
+        if self.offset is not None:
+            body["offset"] = self.offset
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the TimeWindow into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.duration is not None:
+            body["duration"] = self.duration
+        if self.offset is not None:
+            body["offset"] = self.offset
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> TimeWindow:
+        """Deserializes the TimeWindow from a dictionary."""
+        return cls(duration=d.get("duration", None), offset=d.get("offset", None))
+
+
 @dataclass
 class TransitionRequest:
     """For activities, this contains the activity recorded for the action. For comments, this contains
@@ -6256,6 +6538,116 @@ class ExperimentsAPI:
         return UpdateRunResponse.from_dict(res)
 
 
+class FeatureEngineeringAPI:
+    """[description]"""
+
+    def __init__(self, api_client):
+        self._api = api_client
+
+    def create_feature(self, feature: Feature) -> Feature:
+        """Create a Feature.
+
+        :param feature: :class:`Feature`
+          Feature to create.
+
+        :returns: :class:`Feature`
+        """
+        body = feature.as_dict()
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+
+        res = self._api.do("POST", "/api/2.0/feature-engineering/features", body=body, headers=headers)
+        return Feature.from_dict(res)
+
+    def delete_feature(self, full_name: str):
+        """Delete a Feature.
+
+        :param full_name: str
+          Name of the feature to delete.
+
+
+        """
+
+        headers = {
+            "Accept": "application/json",
+        }
+
+        self._api.do("DELETE", f"/api/2.0/feature-engineering/features/{full_name}", headers=headers)
+
+    def get_feature(self, full_name: str) -> Feature:
+        """Get a Feature.
+
+        :param full_name: str
+          Name of the feature to get.
+
+        :returns: :class:`Feature`
+        """
+
+        headers = {
+            "Accept": "application/json",
+        }
+
+        res = self._api.do("GET", f"/api/2.0/feature-engineering/features/{full_name}", headers=headers)
+        return Feature.from_dict(res)
+
+    def list_features(self, *, page_size: Optional[int] = None, page_token: Optional[str] = None) -> Iterator[Feature]:
+        """List Features.
+
+        :param page_size: int (optional)
+          The maximum number of results to return.
+        :param page_token: str (optional)
+          Pagination token to go to the next page based on a previous query.
+
+        :returns: Iterator over :class:`Feature`
+        """
+
+        query = {}
+        if page_size is not None:
+            query["page_size"] = page_size
+        if page_token is not None:
+            query["page_token"] = page_token
+        headers = {
+            "Accept": "application/json",
+        }
+
+        while True:
+            json = self._api.do("GET", "/api/2.0/feature-engineering/features", query=query, headers=headers)
+            if "features" in json:
+                for v in json["features"]:
+                    yield Feature.from_dict(v)
+            if "next_page_token" not in json or not json["next_page_token"]:
+                return
+            query["page_token"] = json["next_page_token"]
+
+    def update_feature(self, full_name: str, feature: Feature, update_mask: str) -> Feature:
+        """Update a Feature.
+
+        :param full_name: str
+          The full three-part name (catalog, schema, name) of the feature.
+        :param feature: :class:`Feature`
+          Feature to update.
+        :param update_mask: str
+          The list of fields to update.
+
+        :returns: :class:`Feature`
+        """
+        body = feature.as_dict()
+        query = {}
+        if update_mask is not None:
+            query["update_mask"] = update_mask
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+
+        res = self._api.do(
+            "PATCH", f"/api/2.0/feature-engineering/features/{full_name}", query=query, body=body, headers=headers
+        )
+        return Feature.from_dict(res)
+
+
 class FeatureStoreAPI:
     """A feature store is a centralized repository that enables data scientists to find and share features. Using
     a feature store also ensures that the code used to compute feature values is the same during model

databricks/sdk/service/oauth2.py

@@ -232,11 +232,11 @@ class FederationPolicy:
     oidc_policy: Optional[OidcFederationPolicy] = None
 
     policy_id: Optional[str] = None
-    """The ID of the federation policy."""
+    """The ID of the federation policy. Output only."""
 
     service_principal_id: Optional[int] = None
-    """The service principal ID that this federation policy applies to. Only set for service principal
-    federation policies."""
+    """The service principal ID that this federation policy applies to. Output only. Only set for
+    service principal federation policies."""
 
     uid: Optional[str] = None
     """Unique, immutable id of the federation policy."""