databricks-sdk 0.154.0__tar.gz → 0.155.0__tar.gz

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Version changelog
 
+## Release v0.155.0 (2026-10-21)
+
+### API Changes
+* Add `sdp_source` field for `databricks.sdk.service.ml.DataSource`.
+* Add `customer_version_label`, `max_dbu` and `reyden_cluster_size_multiplier` fields for `databricks.sdk.service.sql.CreateWarehouseRequest`.
+* Add `customer_version_label`, `max_dbu` and `reyden_cluster_size_multiplier` fields for `databricks.sdk.service.sql.EditWarehouseRequest`.
+* Add `customer_version_label`, `max_dbu` and `reyden_cluster_size_multiplier` fields for `databricks.sdk.service.sql.EndpointInfo`.
+* Add `customer_version_label`, `max_dbu` and `reyden_cluster_size_multiplier` fields for `databricks.sdk.service.sql.GetWarehouseResponse`.
+* [Breaking] Remove `replication_config` field for `databricks.sdk.service.sharing.CreateShare`.
+* [Breaking] Remove `replication_config` field for `databricks.sdk.service.sharing.ShareInfo`.
+
 ## Release v0.154.0 (2026-10-19)
 
 ### API Changes

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-sdk
-Version: 0.154.0
+Version: 0.155.0
 Summary: Databricks SDK for Python (Beta)
 Project-URL: Documentation, https://databricks-sdk-py.readthedocs.io
 Keywords: databricks,sdk

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/config.py

@@ -267,6 +267,11 @@ class Config:
     # When enabled, the SDK will probe the storage proxy and use it if available.
     experimental_files_ext_enable_storage_proxy: bool = False
 
+    # Whether to ask the workspace which endpoints to use for file transfers (GET /api/2.0/fs/endpoints).
+    # When disabled, or when the server cannot answer, the SDK uses a built-in order: Presigned URL, then
+    # Files API 2.0 on the workspace host.
+    experimental_files_ext_enable_endpoints_api: bool = False
+
     def __init__(
         self,
         *,

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/mixins/files.py

@@ -16,6 +16,7 @@ from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from datetime import timedelta
+from enum import Enum
 from io import BytesIO
 from queue import Empty, Full, Queue
 from tempfile import mkstemp
@@ -746,6 +747,35 @@ class _PresignedUrl:
     part_number: int | None = None
 
 
+class _ResolvedEndpoint(Enum):
+    """A file transfer protocol this SDK can speak, identified by the (api, api_version) pair the
+    endpoints API returns."""
+
+    IO_OFFLOAD = "IO_OFFLOAD"
+    FILES_API_V2 = "FILES_API_V2"
+
+
+# Maps the server's (api, api_version) to a protocol compiled into this SDK. Pairs not listed here are
+# skipped, which lets the server advertise newer protocols without breaking older SDKs. IO_OFFLOAD is
+# unversioned during the preview, so a versioned IO_OFFLOAD is newer than this SDK.
+_RECOGNIZED_ENDPOINTS: dict[tuple[str, Optional[str]], _ResolvedEndpoint] = {
+    ("IO_OFFLOAD", None): _ResolvedEndpoint.IO_OFFLOAD,
+    ("FILES_API", "2.0"): _ResolvedEndpoint.FILES_API_V2,
+}
+
+_ENDPOINT_SCHEMES = ("http", "https")
+
+
+@dataclass(frozen=True)
+class _Endpoint:
+    """One entry of the ordered endpoint list a file transfer walks, best first."""
+
+    kind: _ResolvedEndpoint
+    base_url: str
+    """Scheme and hostname as returned by the server, e.g. https://example.cloud.databricks.com."""
+    multipart_upload_supported: bool
+
+
 class _PresignedUrlRequestBuilder:
     """Builds presigned URL requests for file upload operations.
 
@@ -928,6 +958,12 @@ class FilesExt(files.FilesAPI):
     # Timeout in seconds for the storage-proxy health check probe.
     _STORAGE_PROXY_PROBE_TIMEOUT: float = 3.0
 
+    _ENDPOINTS_API_PATH: str = "/api/2.0/fs/endpoints"
+
+    # Endpoint discovery is a single attempt bounded by this timeout: it precedes a transfer, so it
+    # must fail fast to the default order rather than retry and stall the upload or download.
+    _ENDPOINTS_RESOLVE_TIMEOUT_SECONDS: float = 10.0
+
     @dataclass(frozen=True)
     class _UploadContext:
         target_path: str
@@ -956,13 +992,129 @@ class FilesExt(files.FilesAPI):
         self._dp_hostname_available: Optional[bool] = None
         self._cached_storage_proxy_session: Optional[requests.Session] = None
         self._storage_proxy_session_lock = Lock()
+        self._cached_endpoints: Optional[list[_Endpoint]] = None
+        self._endpoints_lock = Lock()
 
-    def _get_hostname(self) -> str:
-        """Returns the hostname for file operations.
+    def _resolve_endpoints(self) -> list[_Endpoint]:
+        """Returns the ordered endpoints to try for a file transfer, best first.
 
-        When storage proxy is enabled, probes it on first call and caches the
-        result. Returns the proxy hostname if available, otherwise the workspace
-        hostname.
+        The server is asked once per client and its answer, including a failure, is cached. Anything
+        short of a usable list falls back to the built-in order, so endpoint discovery never fails a
+        transfer.
+        """
+        if not self._config.experimental_files_ext_enable_endpoints_api:
+            return self._default_endpoints()
+        with self._endpoints_lock:
+            if self._cached_endpoints is None:
+                self._cached_endpoints = self._fetch_endpoints() or self._default_endpoints()
+            return list(self._cached_endpoints)
+
+    def _default_endpoints(self) -> list[_Endpoint]:
+        host = self._config.host
+        return [
+            _Endpoint(_ResolvedEndpoint.IO_OFFLOAD, host, multipart_upload_supported=True),
+            _Endpoint(_ResolvedEndpoint.FILES_API_V2, host, multipart_upload_supported=False),
+        ]
+
+    def _fetch_endpoints(self) -> list[_Endpoint]:
+        """Fetches the endpoint list from the server, keeping only entries this SDK can speak.
+
+        Discovery runs before a transfer, so it is a single short-timeout attempt with no retries: a
+        slow or failing endpoint must fall back to the default order quickly rather than hold up the
+        upload or download.
+        """
+        url = f"{self._config.host}{self._ENDPOINTS_API_PATH}"
+        try:
+            response = self._cloud_provider_session().request(
+                "GET",
+                url,
+                headers={**self._config.authenticate(), "Accept": "application/json"},
+                timeout=self._ENDPOINTS_RESOLVE_TIMEOUT_SECONDS,
+            )
+        except Exception as e:
+            # Broad on purpose: network and auth failures must all fail safe to the default order.
+            _LOG.warning(f"Failed to resolve file endpoints, using the default endpoints: {e}")
+            return []
+
+        # Lets a failure below be traced to the server's request log.
+        request_id = response.headers.get("x-request-id")
+        if response.status_code == 404:
+            # Not a warning: until the server-side rollout completes, some workspaces do not serve the API.
+            _LOG.info(
+                f"Endpoints API is not served for this workspace (request ID {request_id}), using the default endpoints."
+            )
+            return []
+        if response.status_code != 200:
+            _LOG.warning(
+                f"Unexpected status resolving file endpoints ({response.status_code}, request ID {request_id}), "
+                "using the default endpoints."
+            )
+            return []
+        try:
+            body = response.json()
+        except Exception as e:
+            _LOG.warning(
+                f"Failed to parse endpoints response (request ID {request_id}), using the default endpoints: {e}"
+            )
+            return []
+
+        # Proto3 JSON leaves out an empty repeated field.
+        entries = body.get("endpoints", []) if isinstance(body, dict) else None
+        if not isinstance(entries, list):
+            _LOG.warning(
+                f"Unexpected endpoints response (request ID {request_id}), using the default endpoints: {body}"
+            )
+            return []
+        endpoints = []
+        for entry in entries:
+            kind = self._endpoint_kind(entry)
+            if kind is None:
+                # Not a warning: an older SDK is expected to skip entries for APIs added after it shipped.
+                _LOG.info(f"Skipping endpoint this SDK cannot use (request ID {request_id}): {entry}")
+                continue
+            endpoint = self._parse_endpoint(entry, kind)
+            if endpoint is None:
+                _LOG.warning(
+                    f"Skipping {kind.value} endpoint with an unusable address (request ID {request_id}): {entry}"
+                )
+                continue
+            endpoints.append(endpoint)
+        if not endpoints:
+            _LOG.warning(f"No usable endpoints in the response (request ID {request_id}), using the default endpoints.")
+        return endpoints
+
+    @staticmethod
+    def _endpoint_kind(entry) -> Optional[_ResolvedEndpoint]:
+        """Returns the protocol of a listed endpoint, or None if this SDK does not speak it."""
+        if not isinstance(entry, dict):
+            return None
+        api = entry.get("api")
+        api_version = entry.get("api_version")
+        # The entry is unvalidated JSON: an unhashable value such as a list would make the lookup below raise
+        # instead of skipping the entry. IO_OFFLOAD entries have no api_version, so None is allowed.
+        if not isinstance(api, str) or not (api_version is None or isinstance(api_version, str)):
+            return None
+        return _RECOGNIZED_ENDPOINTS.get((api, api_version))
+
+    @staticmethod
+    def _parse_endpoint(entry: dict, kind: _ResolvedEndpoint) -> Optional[_Endpoint]:
+        scheme = entry.get("scheme")
+        hostname = entry.get("hostname")
+        if scheme not in _ENDPOINT_SCHEMES or not isinstance(hostname, str) or not hostname:
+            return None
+        # Mirrors the proto getter: an unset optional bool reads as false.
+        multipart_upload_supported = entry.get("multipart_upload_supported") is True
+        return _Endpoint(kind, f"{scheme}://{hostname}", multipart_upload_supported)
+
+    def _active_endpoint(self) -> _Endpoint:
+        """The endpoint used for bulk file transfer.
+
+        Selected by the storage-proxy probe: the storage proxy (a direct Files API path) when the probe
+        finds it available, otherwise the offload (presigned) path on the workspace host. The probe runs
+        once and its result is cached.
+
+        This is the single-endpoint selection; PR wiring the endpoints API replaces it with a walk over
+        the server-resolved list.
         """
         if self._config.experimental_files_ext_enable_storage_proxy:
             if self._dp_hostname_available is None:
@@ -972,19 +1124,28 @@ class FilesExt(files.FilesAPI):
                 else:
                     _LOG.info("Storage proxy is not available, will use presigned URLs.")
             if self._dp_hostname_available:
-                return self._STORAGE_PROXY_HOSTNAME
-        return self._config.host
+                return _Endpoint(
+                    _ResolvedEndpoint.FILES_API_V2, self._STORAGE_PROXY_HOSTNAME, multipart_upload_supported=True
+                )
+        return _Endpoint(_ResolvedEndpoint.IO_OFFLOAD, self._config.host, multipart_upload_supported=True)
+
+    def _get_hostname(self) -> str:
+        """Returns the scheme+hostname for file operations, e.g. https://example.cloud.databricks.com."""
+        return self._active_endpoint().base_url
 
     def _create_request_builder(self):
-        """Creates the appropriate request builder for file operations.
+        """Creates the request builder for the active endpoint."""
+        return self._request_builder_for(self._active_endpoint())
+
+    def _request_builder_for(self, endpoint: _Endpoint):
+        """Maps an endpoint to the builder that speaks its protocol.
 
-        Returns a storage proxy builder when the proxy is available,
-        otherwise a presigned URL builder.
+        The offload path coordinates presigned URLs through the workspace API; the direct Files API path
+        builds URLs straight at the endpoint host.
         """
-        hostname = self._get_hostname()
-        if self._config.experimental_files_ext_enable_storage_proxy and self._dp_hostname_available:
-            return _StorageProxyRequestBuilder(hostname)
-        return _PresignedUrlRequestBuilder(self._api, hostname)
+        if endpoint.kind == _ResolvedEndpoint.IO_OFFLOAD:
+            return _PresignedUrlRequestBuilder(self._api, endpoint.base_url)
+        return _StorageProxyRequestBuilder(endpoint.base_url)
 
     def _resolve_workspace_id(self) -> Optional[str]:
         """Resolves the workspace ID by querying the SCIM Me endpoint."""

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/service/billing.py

@@ -193,8 +193,15 @@ class BudgetConfiguration:
     """Databricks account ID."""
 
     alert_configurations: Optional[List[AlertConfiguration]] = None
-    """Alerts to configure when this budget is in a triggered state. Budgets must have exactly one
-    alert configuration."""
+    """Alerts to configure when this budget is in a triggered state. Budgets must have at least one
+    alert configuration. A budget can have up to four shared alerts per time period, each with a
+    unique ``quantity_threshold``. Budgets with ``resource_type`` set to
+    ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY`` can also use per-user alerts, with optional
+    ``principal_overrides``. These budgets can have at most one per-user alert per time period, and
+    can use shared alerts, a per-user alert, or both. The ``BLOCK_USAGE`` action is only supported
+    on budgets with ``resource_type`` set to ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY``. At most one
+    shared alert per time period can have a ``BLOCK_USAGE`` action, and it must be the shared alert
+    with the highest threshold in that period."""
 
     budget_configuration_id: Optional[str] = None
     """Databricks budget configuration ID."""
@@ -504,8 +511,15 @@ class CreateBudgetConfigurationBudget:
     """Databricks account ID."""
 
     alert_configurations: Optional[List[CreateBudgetConfigurationBudgetAlertConfigurations]] = None
-    """Alerts to configure when this budget is in a triggered state. Budgets must have exactly one
-    alert configuration."""
+    """Alerts to configure when this budget is in a triggered state. Budgets must have at least one
+    alert configuration. A budget can have up to four shared alerts per time period, each with a
+    unique ``quantity_threshold``. Budgets with ``resource_type`` set to
+    ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY`` can also use per-user alerts, with optional
+    ``principal_overrides``. These budgets can have at most one per-user alert per time period, and
+    can use shared alerts, a per-user alert, or both. The ``BLOCK_USAGE`` action is only supported
+    on budgets with ``resource_type`` set to ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY``. At most one
+    shared alert per time period can have a ``BLOCK_USAGE`` action, and it must be the shared alert
+    with the highest threshold in that period."""
 
     display_name: Optional[str] = None
     """Human-readable name of budget configuration. Max Length: 128"""
@@ -1470,8 +1484,15 @@ class UpdateBudgetConfigurationBudget:
     """Databricks account ID."""
 
     alert_configurations: Optional[List[AlertConfiguration]] = None
-    """Alerts to configure when this budget is in a triggered state. Budgets must have exactly one
-    alert configuration."""
+    """Alerts to configure when this budget is in a triggered state. Budgets must have at least one
+    alert configuration. A budget can have up to four shared alerts per time period, each with a
+    unique ``quantity_threshold``. Budgets with ``resource_type`` set to
+    ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY`` can also use per-user alerts, with optional
+    ``principal_overrides``. These budgets can have at most one per-user alert per time period, and
+    can use shared alerts, a per-user alert, or both. The ``BLOCK_USAGE`` action is only supported
+    on budgets with ``resource_type`` set to ``BUDGET_RESOURCE_TYPE_UNITY_AI_GATEWAY``. At most one
+    shared alert per time period can have a ``BLOCK_USAGE`` action, and it must be the shared alert
+    with the highest threshold in that period."""
 
     budget_configuration_id: Optional[str] = None
     """Databricks budget configuration ID."""

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/service/bundledeployments.py

@@ -559,10 +559,11 @@ class Operation:
     rejected with ABORTED so the caller can re-read and retry."""
 
     state: Optional[str] = None
-    """Serialized local config state after the operation. Its presence records whether the resource
-    still exists. Explicitly clearing state removes the resource; omitting it from the update mask
-    leaves resource state unchanged. State is stored only on Resources. UpdateOperation accepts
-    state but omits it from its response, as do subsequent GetOperation and ListOperations.
+    """Serialized local config state after the operation. Its presence records whether the resource is
+    still tracked by the deployment. Explicitly clearing state removes the resource from the
+    deployment; omitting it from the update mask leaves resource state unchanged. State is stored
+    only on Resources. UpdateOperation accepts state but omits it from its response, as do
+    subsequent GetOperation and ListOperations.
     
     Mutable: may be updated after creation via UpdateOperation. When updating, the caller must echo
     the last-observed ``sequence_id`` as a concurrency precondition.
@@ -1529,8 +1530,8 @@ class BundleDeploymentsAPI:
         value it last observed, and the server rejects the update with ``ABORTED`` if the operation has been
         modified since. On success the server increments ``sequence_id``; updates to ``state``,
         ``resource_id``, and ``dashboard_metadata`` are mirrored onto the corresponding deployment-level
-        resource. Listing ``state`` in ``update_mask`` with no value clears it, which removes the resource, so
-        a delete that is retried until it succeeds must clear ``state``. The parent version must be in
+        resource. Listing ``state`` in ``update_mask`` with no value clears it, which removes the tracked
+        resource, so a delete or unbind that succeeds must clear ``state``. The parent version must be in
         progress, and after the update is applied a succeeded operation cannot carry an ``error_message``. See
         the ``state`` and ``resource_id`` fields for the rest.
 

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/service/catalog.py

@@ -8051,7 +8051,7 @@ class ModelProviderServiceConfigCustomProviderApiKeyHeaderAuth:
 
 @dataclass
 class ModelProviderServiceConfigCustomProviderConfig:
-    """Custom OpenAI-compatible provider configuration with bearer-token authentication."""
+    """Custom OpenAI-compatible provider configuration."""
 
     direct: Optional[ModelProviderServiceConfigCustomProviderDirectConfig] = None
     """Endpoint and authentication configuration for the custom provider."""
@@ -14464,8 +14464,16 @@ class AiGatewayAPI:
 
     def create_skill(self, skill: Skill, parent: str, skill_id: str) -> Skill:
         """Creates a skill in a Unity Catalog schema and provisions its managed bundle storage. Specify its name
-        in ``skill_id``. The request contains an optional comment but no bundle bytes. Upload bundle files
-        through the Files API, then call FinalizeSkill.
+        in ``skill_id``. The request contains an optional comment but no bundle bytes.
+
+        To publish a skill:
+
+        1. Call CreateSkill to create the skill in Unity Catalog.
+        2. Upload every bundle file, including ``SKILL.md`` at the bundle root, with the Files API: ``PUT
+           /api/2.0/fs/files/Skills/{catalog}/{schema}/{skill}/{file_path}``. The CreateSkill caller owns the
+           skill and can upload. Other callers need ``READ_VOLUME`` and ``WRITE_VOLUME`` on the skill.
+        3. Call FinalizeSkill to validate ``SKILL.md`` and set the skill's ``bundle_name`` and ``description``
+           from its frontmatter ``name`` and ``description``.
 
         You must be the owner of the parent schema or have ``CREATE_VOLUME`` and ``USE_SCHEMA`` on it, plus
         ``USE_CATALOG`` on the parent catalog.
@@ -14651,12 +14659,13 @@ class AiGatewayAPI:
         self._api.do("DELETE", f"/api/2.1/unity-catalog/{name}", query=query, headers=headers)
 
     def finalize_skill(self, name: str) -> Skill:
-        """Finalizes a skill after its bundle is uploaded. This method reads SKILL.md through the Files API using
-        the caller's authorization. Its YAML frontmatter must contain an agentskills.io-compliant ``name`` and
-        a nonblank ``description`` within the configured UTF-8 byte limit. On success, it replaces
-        ``bundle_name`` and ``description``; refreshes ``finalize_time``, ``update_time``, and ``updated_by``;
-        and returns the updated skill. ``comment`` is preserved. Re-finalization uses the latest SKILL.md and
-        is last-write-wins without an etag precondition. Validation failures do not change metadata.
+        """Finalizes a skill after its bundle files are uploaded or changed through the Files API. Call it again
+        after every bundle change. This method reads SKILL.md through the Files API using the caller's
+        authorization. Its YAML frontmatter must contain an agentskills.io-compliant ``name`` and a nonblank
+        ``description`` within the configured UTF-8 byte limit. On success, it replaces ``bundle_name`` and
+        ``description``; refreshes ``finalize_time``, ``update_time``, and ``updated_by``; and returns the
+        updated skill. ``comment`` is preserved. Re-finalization uses the latest SKILL.md and is
+        last-write-wins without an etag precondition. Validation failures do not change metadata.
 
         You must be the owner of the skill or have ``READ_VOLUME`` on it, plus ``USE_CATALOG`` on the parent
         catalog and ``USE_SCHEMA`` on the parent schema.
@@ -14819,7 +14828,7 @@ class AiGatewayAPI:
         returned.
 
         :param page_size: int (optional)
-          Maximum number of MCP services to return. Defaults to 100 when unset or 0; the maximum is 200. Use
+          Maximum number of MCP services to return. Defaults to 200 when unset or 0; the maximum is 200. Use
           ``page_token`` to retrieve additional pages.
         :param page_token: str (optional)
           Opaque pagination token from the previous response.
@@ -14877,7 +14886,7 @@ class AiGatewayAPI:
         ``MANAGE``) are returned.
 
         :param page_size: int (optional)
-          Maximum number of provider services to return. Defaults to 100 when unset or 0; the maximum is 200.
+          Maximum number of provider services to return. Defaults to 200 when unset or 0; the maximum is 200.
           Use ``page_token`` to retrieve additional pages.
         :param page_token: str (optional)
           Opaque pagination token from the previous response.
@@ -14935,7 +14944,7 @@ class AiGatewayAPI:
         returned.
 
         :param page_size: int (optional)
-          Maximum number of model services to return. Defaults to 100 when unset or 0; the maximum is 200. Use
+          Maximum number of model services to return. Defaults to 200 when unset or 0; the maximum is 200. Use
           ``page_token`` to retrieve additional pages.
         :param page_token: str (optional)
           Opaque pagination token from the previous response.
@@ -14993,7 +15002,7 @@ class AiGatewayAPI:
           Required: skill listing is schema-scoped, so ``parent`` must be set; an unset or empty ``parent`` is
           rejected with INVALID_PARAMETER_VALUE.
         :param page_size: int (optional)
-          Maximum number of skills to return. Defaults to 100 when unset or 0; the maximum is 200. Use
+          Maximum number of skills to return. Defaults to 200 when unset or 0; the maximum is 200. Use
           ``page_token`` to retrieve additional pages.
         :param page_token: str (optional)
           Opaque pagination token from a previous request.
@@ -15201,8 +15210,13 @@ class AiGatewayAPI:
         conditional on the skill not having changed since it was read. Bundle files, grants, tags, and
         ownership are unchanged.
 
-        You must be the owner of the skill or have ``MANAGE`` on it, plus ``USE_CATALOG`` on the parent
-        catalog and ``USE_SCHEMA`` on the parent schema.
+        To update the bundle, add, overwrite (``overwrite=true``), or delete files under
+        ``/Skills/{catalog}/{schema}/{skill}/`` with the Files API, then call FinalizeSkill. Changing bundle
+        files requires being the owner of the skill or having ``READ_VOLUME`` and ``WRITE_VOLUME`` on it;
+        ``MANAGE`` does not grant access to bundle files.
+
+        To call UpdateSkill, you must be the owner of the skill or have ``MANAGE`` on it, plus ``USE_CATALOG``
+        on the parent catalog and ``USE_SCHEMA`` on the parent schema.
 
         :param name: str
           Resource name of the skill. Format: ``skills/{catalog}.{schema}.{skill}``. Each ``{...}`` component

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/service/ml.py

@@ -1240,6 +1240,9 @@ class DataSource:
     request_source: Optional[RequestSource] = None
     """A request-time data source."""
 
+    sdp_source: Optional[SdpSource] = None
+    """A Spark Declarative Pipeline data source."""
+
     stream_source: Optional[StreamSource] = None
     """A Stream data source."""
 
@@ -1256,6 +1259,8 @@ class DataSource:
             body["lateness"] = self.lateness.as_dict()
         if self.request_source:
             body["request_source"] = self.request_source.as_dict()
+        if self.sdp_source:
+            body["sdp_source"] = self.sdp_source.as_dict()
         if self.stream_source:
             body["stream_source"] = self.stream_source.as_dict()
         return body
@@ -1273,6 +1278,8 @@ class DataSource:
             body["lateness"] = self.lateness
         if self.request_source:
             body["request_source"] = self.request_source
+        if self.sdp_source:
+            body["sdp_source"] = self.sdp_source
         if self.stream_source:
             body["stream_source"] = self.stream_source
         return body
@@ -1286,6 +1293,7 @@ class DataSource:
             kafka_source=_from_dict(d, "kafka_source", KafkaSource),
             lateness=_from_dict(d, "lateness", SourceLateness),
             request_source=_from_dict(d, "request_source", RequestSource),
+            sdp_source=_from_dict(d, "sdp_source", SdpSource),
             stream_source=_from_dict(d, "stream_source", StreamSource),
         )
 
@@ -8144,6 +8152,106 @@ class SchemaRegistryConfig:
         )
 
 
+@dataclass
+class SdpSource:
+    """A Spark Declarative Pipeline (SDP) data source."""
+
+    dataframe_schema: Optional[str] = None
+    """Schema of the resulting dataframe after transformations, in Spark StructType JSON format (from
+    df.schema.json()). Required if transformation_sql is specified. Example:
+    {"type":"struct","fields":[{"name":"col_a","type":"integer","nullable":true,"metadata":{}},{"name":"col_c","type":"integer","nullable":true,"metadata":{}}]}"""
+
+    filter_condition: Optional[str] = None
+    """Single WHERE clause to filter the pipeline's output table before applying transformations. Will
+    be row-wise evaluated, so should only include conditionals and projections."""
+
+    input_tables: Optional[List[str]] = None
+    """The full three-part names of the tables the pipeline reads from. Required to materialize this
+    feature with a TableTrigger, which starts a run when these tables update. Each table must exist
+    when the feature is created. After the pipeline runs, Feature Store checks that it reads from
+    these tables, and the run fails if it doesn't. At most 100 tables."""
+
+    output_table: Optional[str] = None
+    """The full three-part (catalog, schema, table) name of the pipeline's output table to read from.
+    The table doesn't need to exist until the pipeline first runs. Required."""
+
+    pipeline_id: Optional[str] = None
+    """The ID of the Spark Declarative Pipeline that Feature Store runs to produce this source, before
+    each materialization or backfill. The pipeline must already exist when the feature is created,
+    and you need CAN_RUN, CAN_MANAGE, or owner permission on it. It doesn't need to have run yet.
+    Deleting the feature doesn't delete the pipeline. If the pipeline is deleted, this field reads
+    back empty. Required."""
+
+    pipeline_state: Optional[SdpSourceState] = None
+    """State tracking pipeline execution and output table validation. PENDING until the pipeline has
+    run at least once and created the output table; then RESOLVED on success, or ERROR on failure."""
+
+    transformation_sql: Optional[str] = None
+    """A single SQL SELECT expression applied after filter_condition. Should contain all the columns
+    needed (eg. "SELECT *, col_a + col_b AS col_c FROM x.y.z WHERE col_a > 0" would have
+    ``transformation_sql`` "*, col_a + col_b AS col_c") If transformation_sql is not provided, all
+    columns of the pipeline's output table are present in the DataSource dataframe."""
+
+    def as_dict(self) -> dict:
+        """Serializes the SdpSource into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.dataframe_schema is not None:
+            body["dataframe_schema"] = self.dataframe_schema
+        if self.filter_condition is not None:
+            body["filter_condition"] = self.filter_condition
+        if self.input_tables:
+            body["input_tables"] = [v for v in self.input_tables]
+        if self.output_table is not None:
+            body["output_table"] = self.output_table
+        if self.pipeline_id is not None:
+            body["pipeline_id"] = self.pipeline_id
+        if self.pipeline_state is not None:
+            body["pipeline_state"] = self.pipeline_state.value
+        if self.transformation_sql is not None:
+            body["transformation_sql"] = self.transformation_sql
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the SdpSource into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.dataframe_schema is not None:
+            body["dataframe_schema"] = self.dataframe_schema
+        if self.filter_condition is not None:
+            body["filter_condition"] = self.filter_condition
+        if self.input_tables:
+            body["input_tables"] = self.input_tables
+        if self.output_table is not None:
+            body["output_table"] = self.output_table
+        if self.pipeline_id is not None:
+            body["pipeline_id"] = self.pipeline_id
+        if self.pipeline_state is not None:
+            body["pipeline_state"] = self.pipeline_state
+        if self.transformation_sql is not None:
+            body["transformation_sql"] = self.transformation_sql
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> SdpSource:
+        """Deserializes the SdpSource from a dictionary."""
+        return cls(
+            dataframe_schema=d.get("dataframe_schema", None),
+            filter_condition=d.get("filter_condition", None),
+            input_tables=d.get("input_tables", None),
+            output_table=d.get("output_table", None),
+            pipeline_id=d.get("pipeline_id", None),
+            pipeline_state=_enum(d, "pipeline_state", SdpSourceState),
+            transformation_sql=d.get("transformation_sql", None),
+        )
+
+
+class SdpSourceState(Enum):
+    """State of the SDP source relative to pipeline execution and output table validation."""
+
+    ERROR = "ERROR"
+    PENDING = "PENDING"
+    RESOLVED = "RESOLVED"
+
+
 @dataclass
 class SearchExperimentsResponse:
     experiments: Optional[List[Experiment]] = None

{databricks_sdk-0.154.0 → databricks_sdk-0.155.0}/databricks/sdk/service/pipelines.py

@@ -6652,8 +6652,8 @@ class PipelinesAPI:
           optional. The default value is 25. The maximum value is 100. An error is returned if the value of
           max_results is greater than 100.
         :param order_by: List[str] (optional)
-          A list of strings specifying the order of results. Supported order_by fields are id and name. The
-          default is id asc. This field is optional.
+          A list of strings specifying the order of results. The supported order_by field is id. The default
+          is id asc. This field is optional.
         :param page_token: str (optional)
           Page token returned by previous call