databricks-sdk 0.19.1__tar.gz → 0.21.0__tar.gz

{databricks-sdk-0.19.1/databricks_sdk.egg-info → databricks-sdk-0.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: databricks-sdk
-Version: 0.19.1
+Version: 0.21.0
 Summary: Databricks SDK for Python (Beta)
 Home-page: https://databricks-sdk-py.readthedocs.io
 Author: Serge Smertin
@@ -34,6 +34,7 @@ License-File: NOTICE
 [![databricks-sdk](https://snyk.io/advisor/python/databricks-sdk/badge.svg)](https://snyk.io/advisor/python/databricks-sdk)
 ![PyPI](https://img.shields.io/pypi/v/databricks-sdk)
 [![codecov](https://codecov.io/gh/databricks/databricks-sdk-py/branch/main/graph/badge.svg?token=GU63K7WDBE)](https://codecov.io/gh/databricks/databricks-sdk-py)
+[![lines of code](https://tokei.rs/b1/github/databricks/databricks-sdk-py)]([https://codecov.io/github/databricks/databricks-sdk-py](https://github.com/databricks/databricks-sdk-py))
 
 [Beta](https://docs.databricks.com/release-notes/release-types.html): This SDK is supported for production use cases, 
 but we do expect future releases to have some interface changes; see [Interface stability](#interface-stability). 

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/README.md

@@ -5,6 +5,7 @@
 [![databricks-sdk](https://snyk.io/advisor/python/databricks-sdk/badge.svg)](https://snyk.io/advisor/python/databricks-sdk)
 ![PyPI](https://img.shields.io/pypi/v/databricks-sdk)
 [![codecov](https://codecov.io/gh/databricks/databricks-sdk-py/branch/main/graph/badge.svg?token=GU63K7WDBE)](https://codecov.io/gh/databricks/databricks-sdk-py)
+[![lines of code](https://tokei.rs/b1/github/databricks/databricks-sdk-py)]([https://codecov.io/github/databricks/databricks-sdk-py](https://github.com/databricks/databricks-sdk-py))
 
 [Beta](https://docs.databricks.com/release-notes/release-types.html): This SDK is supported for production use cases, 
 but we do expect future releases to have some interface changes; see [Interface stability](#interface-stability). 

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/__init__.py

@@ -15,6 +15,7 @@ from databricks.sdk.service.catalog import (AccountMetastoreAssignmentsAPI,
                                             ExternalLocationsAPI, FunctionsAPI,
                                             GrantsAPI, LakehouseMonitorsAPI,
                                             MetastoresAPI, ModelVersionsAPI,
+                                            OnlineTablesAPI,
                                             RegisteredModelsAPI, SchemasAPI,
                                             StorageCredentialsAPI,
                                             SystemSchemasAPI,
@@ -33,9 +34,9 @@ from databricks.sdk.service.iam import (AccountAccessControlAPI,
                                         AccountGroupsAPI,
                                         AccountServicePrincipalsAPI,
                                         AccountUsersAPI, CurrentUserAPI,
-                                        GroupsAPI, PermissionsAPI,
-                                        ServicePrincipalsAPI, UsersAPI,
-                                        WorkspaceAssignmentAPI)
+                                        GroupsAPI, PermissionMigrationAPI,
+                                        PermissionsAPI, ServicePrincipalsAPI,
+                                        UsersAPI, WorkspaceAssignmentAPI)
 from databricks.sdk.service.jobs import JobsAPI
 from databricks.sdk.service.ml import ExperimentsAPI, ModelRegistryAPI
 from databricks.sdk.service.oauth2 import (CustomAppIntegrationAPI,
@@ -51,9 +52,17 @@ from databricks.sdk.service.provisioning import (CredentialsAPI,
 from databricks.sdk.service.serving import AppsAPI, ServingEndpointsAPI
 from databricks.sdk.service.settings import (AccountIpAccessListsAPI,
                                              AccountSettingsAPI,
+                                             AutomaticClusterUpdateAPI,
                                              CredentialsManagerAPI,
+                                             CspEnablementAccountAPI,
+                                             CspEnablementAPI,
+                                             DefaultNamespaceAPI,
+                                             EsmEnablementAccountAPI,
+                                             EsmEnablementAPI,
                                              IpAccessListsAPI,
                                              NetworkConnectivityAPI,
+                                             PersonalComputeAPI,
+                                             RestrictWorkspaceAdminsAPI,
                                              SettingsAPI, TokenManagementAPI,
                                              TokensAPI, WorkspaceConfAPI)
 from databricks.sdk.service.sharing import (CleanRoomsAPI, ProvidersAPI,
@@ -180,6 +189,8 @@ class WorkspaceClient:
         self._metastores = MetastoresAPI(self._api_client)
         self._model_registry = ModelRegistryAPI(self._api_client)
         self._model_versions = ModelVersionsAPI(self._api_client)
+        self._online_tables = OnlineTablesAPI(self._api_client)
+        self._permission_migration = PermissionMigrationAPI(self._api_client)
         self._permissions = PermissionsAPI(self._api_client)
         self._pipelines = PipelinesAPI(self._api_client)
         self._policy_families = PolicyFamiliesAPI(self._api_client)
@@ -322,7 +333,7 @@ class WorkspaceClient:
 
     @property
     def files(self) -> FilesAPI:
-        """The Files API allows you to read, write, and delete files and directories in Unity Catalog volumes."""
+        """The Files API is a standard HTTP API that allows you to read, write, list, and delete files and directories by referring to their URI."""
         return self._files
 
     @property
@@ -400,6 +411,16 @@ class WorkspaceClient:
         """Databricks provides a hosted version of MLflow Model Registry in Unity Catalog."""
         return self._model_versions
 
+    @property
+    def online_tables(self) -> OnlineTablesAPI:
+        """Online tables provide lower latency and higher QPS access to data from Delta tables."""
+        return self._online_tables
+
+    @property
+    def permission_migration(self) -> PermissionMigrationAPI:
+        """This spec contains undocumented permission migration APIs used in https://github.com/databrickslabs/ucx."""
+        return self._permission_migration
+
     @property
     def permissions(self) -> PermissionsAPI:
         """Permissions API are used to create read, write, edit, update and manage access for various users on different objects and endpoints."""
@@ -477,7 +498,7 @@ class WorkspaceClient:
 
     @property
     def settings(self) -> SettingsAPI:
-        """The default namespace setting API allows users to configure the default namespace for a Databricks workspace."""
+        """Workspace Settings API allows users to manage settings at the workspace level."""
         return self._settings
 
     @property
@@ -754,7 +775,7 @@ class AccountClient:
 
     @property
     def settings(self) -> AccountSettingsAPI:
-        """The Personal Compute enablement setting lets you control which users can use the Personal Compute default policy to create compute resources."""
+        """Accounts Settings API allows users to manage settings at the account level."""
         return self._settings
 
     @property
@@ -813,6 +834,7 @@ class AccountClient:
         config.host = config.environment.deployment_url(workspace.deployment_name)
         config.azure_workspace_resource_id = azure.get_azure_resource_id(workspace)
         config.account_id = None
+        config.init_auth()
         return WorkspaceClient(config=config)
 
     def __repr__(self):

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/_widgets/__init__.py

@@ -13,11 +13,11 @@ class WidgetUtils(ABC):
     def _get(self, name: str) -> str:
         pass
 
-    def getArgument(self, name: str, default_value: typing.Optional[str] = None):
+    def getArgument(self, name: str, defaultValue: typing.Optional[str] = None):
         try:
             return self.get(name)
         except Exception:
-            return default_value
+            return defaultValue
 
     def remove(self, name: str):
         self._remove(name)

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/config.py

@@ -87,6 +87,7 @@ class Config:
                  product_version="0.0.0",
                  clock: Clock = None,
                  **kwargs):
+        self._header_factory = None
         self._inner = {}
         self._user_agent_other_info = []
         self._credentials_provider = credentials_provider if credentials_provider else DefaultCredentials()
@@ -100,7 +101,7 @@ class Config:
             self._known_file_config_loader()
             self._fix_host_if_needed()
             self._validate()
-            self._init_auth()
+            self.init_auth()
             self._product = product
             self._product_version = product_version
         except ValueError as e:
@@ -436,7 +437,7 @@ class Config:
         names = " and ".join(sorted(auths_used))
         raise ValueError(f'validate: more than one authorization method configured: {names}')
 
-    def _init_auth(self):
+    def init_auth(self):
         try:
             self._header_factory = self._credentials_provider(self)
             self.auth_type = self._credentials_provider.auth_type()

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/core.py

@@ -135,11 +135,13 @@ class ApiClient:
                                             raw=raw,
                                             files=files,
                                             data=data)
-        if raw:
-            return StreamingResponse(response)
+
         resp = dict()
         for header in response_headers if response_headers else []:
             resp[header] = response.headers.get(Casing.to_header_case(header))
+        if raw:
+            resp["contents"] = StreamingResponse(response)
+            return resp
         if not len(response.content):
             return resp
 

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/mixins/workspace.py

@@ -102,4 +102,5 @@ class WorkspaceExt(WorkspaceAPI):
         """
         query = {'path': path, 'direct_download': 'true'}
         if format: query['format'] = format.value
-        return self._api.do('GET', '/api/2.0/workspace/export', query=query, raw=True)
+        response = self._api.do('GET', '/api/2.0/workspace/export', query=query, raw=True)
+        return response["contents"]

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/oauth.py

@@ -366,7 +366,7 @@ class OAuthClient:
 
         config = Config(host=host, credentials_provider=noop_credentials)
         if not scopes:
-            scopes = ['offline_access', 'clusters', 'sql']
+            scopes = ['all-apis']
         if config.is_azure:
             # Azure AD only supports full access to Azure Databricks.
             scopes = [f'{config.effective_azure_login_app_id}/user_impersonation', 'offline_access']

databricks-sdk-0.21.0/databricks/sdk/runtime/__init__.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import logging
+from typing import Dict, Optional, Union, cast
+
+logger = logging.getLogger('databricks.sdk')
+is_local_implementation = True
+
+# All objects that are injected into the Notebook's user namespace should also be made
+# available to be imported from databricks.sdk.runtime.globals. This import can be used
+# in Python modules so users can access these objects from Files more easily.
+dbruntime_objects = [
+    "display", "displayHTML", "dbutils", "table", "sql", "udf", "getArgument", "sc", "sqlContext", "spark"
+]
+
+# DO NOT MOVE THE TRY-CATCH BLOCK BELOW AND DO NOT ADD THINGS BEFORE IT! WILL MAKE TEST FAIL.
+try:
+    # We don't want to expose additional entity to user namespace, so
+    # a workaround here for exposing required information in notebook environment
+    from dbruntime.sdk_credential_provider import init_runtime_native_auth
+    logger.debug('runtime SDK credential provider available')
+    dbruntime_objects.append("init_runtime_native_auth")
+except ImportError:
+    init_runtime_native_auth = None
+
+globals()["init_runtime_native_auth"] = init_runtime_native_auth
+
+
+def init_runtime_repl_auth():
+    try:
+        from dbruntime.databricks_repl_context import get_context
+        ctx = get_context()
+        if ctx is None:
+            logger.debug('Empty REPL context returned, skipping runtime auth')
+            return None, None
+        if ctx.workspaceUrl is None:
+            logger.debug('Workspace URL is not available, skipping runtime auth')
+            return None, None
+        host = f'https://{ctx.workspaceUrl}'
+
+        def inner() -> Dict[str, str]:
+            ctx = get_context()
+            return {'Authorization': f'Bearer {ctx.apiToken}'}
+
+        return host, inner
+    except ImportError:
+        return None, None
+
+
+def init_runtime_legacy_auth():
+    try:
+        import IPython
+        ip_shell = IPython.get_ipython()
+        if ip_shell is None:
+            return None, None
+        global_ns = ip_shell.ns_table["user_global"]
+        if 'dbutils' not in global_ns:
+            return None, None
+        dbutils = global_ns["dbutils"].notebook.entry_point.getDbutils()
+        if dbutils is None:
+            return None, None
+        ctx = dbutils.notebook().getContext()
+        if ctx is None:
+            return None, None
+        host = getattr(ctx, 'apiUrl')().get()
+
+        def inner() -> Dict[str, str]:
+            ctx = dbutils.notebook().getContext()
+            return {'Authorization': f'Bearer {getattr(ctx, "apiToken")().get()}'}
+
+        return host, inner
+    except ImportError:
+        return None, None
+
+
+try:
+    # Internal implementation
+    # Separated from above for backward compatibility
+    from dbruntime import UserNamespaceInitializer
+
+    userNamespaceGlobals = UserNamespaceInitializer.getOrCreate().get_namespace_globals()
+    _globals = globals()
+    for var in dbruntime_objects:
+        if var not in userNamespaceGlobals:
+            continue
+        _globals[var] = userNamespaceGlobals[var]
+    is_local_implementation = False
+except ImportError:
+    # OSS implementation
+    is_local_implementation = True
+
+    for var in dbruntime_objects:
+        globals()[var] = None
+
+    # The next few try-except blocks are for initialising globals in a best effort
+    # mannaer. We separate them to try to get as many of them working as possible
+    try:
+        # We expect this to fail and only do this for providing types
+        from pyspark.sql.context import SQLContext
+        sqlContext: SQLContext = None # type: ignore
+        table = sqlContext.table
+    except Exception as e:
+        logging.debug(f"Failed to initialize globals 'sqlContext' and 'table', continuing. Cause: {e}")
+
+    try:
+        from pyspark.sql.functions import udf  # type: ignore
+    except ImportError as e:
+        logging.debug(f"Failed to initialise udf global: {e}")
+
+    try:
+        from databricks.connect import DatabricksSession  # type: ignore
+        spark = DatabricksSession.builder.getOrCreate()
+        sql = spark.sql # type: ignore
+    except Exception as e:
+        # We are ignoring all failures here because user might want to initialize
+        # spark session themselves and we don't want to interfere with that
+        logging.debug(f"Failed to initialize globals 'spark' and 'sql', continuing. Cause: {e}")
+
+    try:
+        # We expect this to fail locally since dbconnect does not support sparkcontext. This is just for typing
+        sc = spark.sparkContext
+    except Exception as e:
+        logging.debug(f"Failed to initialize global 'sc', continuing. Cause: {e}")
+
+    def display(input=None, *args, **kwargs) -> None: # type: ignore
+        """
+        Display plots or data.
+        Display plot:
+                        - display() # no-op
+                        - display(matplotlib.figure.Figure)
+        Display dataset:
+                        - display(spark.DataFrame)
+                        - display(list) # if list can be converted to DataFrame, e.g., list of named tuples
+                        - display(pandas.DataFrame)
+                        - display(koalas.DataFrame)
+                        - display(pyspark.pandas.DataFrame)
+        Display any other value that has a _repr_html_() method
+        For Spark 2.0 and 2.1:
+                        - display(DataFrame, streamName='optional', trigger=optional pyspark.sql.streaming.Trigger,
+                                                        checkpointLocation='optional')
+        For Spark 2.2+:
+                        - display(DataFrame, streamName='optional', trigger=optional interval like '1 second',
+                                                        checkpointLocation='optional')
+        """
+        # Import inside the function so that imports are only triggered on usage.
+        from IPython import display as IPDisplay
+        return IPDisplay.display(input, *args, **kwargs) # type: ignore
+
+    def displayHTML(html) -> None: # type: ignore
+        """
+        Display HTML data.
+        Parameters
+        ----------
+        data : URL or HTML string
+                        If data is a URL, display the resource at that URL, the resource is loaded dynamically by the browser.
+                        Otherwise data should be the HTML to be displayed.
+        See also:
+        IPython.display.HTML
+        IPython.display.display_html
+        """
+        # Import inside the function so that imports are only triggered on usage.
+        from IPython import display as IPDisplay
+        return IPDisplay.display_html(html, raw=True) # type: ignore
+
+    # We want to propagate the error in initialising dbutils because this is a core
+    # functionality of the sdk
+    from databricks.sdk.dbutils import RemoteDbUtils
+
+    from . import dbutils_stub
+    dbutils_type = Union[dbutils_stub.dbutils, RemoteDbUtils]
+
+    dbutils = RemoteDbUtils()
+    dbutils = cast(dbutils_type, dbutils)
+
+    # We do this to prevent importing widgets implementation prematurely
+    # The widget import should prompt users to use the implementation
+    # which has ipywidget support.
+    def getArgument(name: str, defaultValue: Optional[str] = None):
+        return dbutils.widgets.getArgument(name, defaultValue)
+
+
+__all__ = dbruntime_objects

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/runtime/dbutils_stub.py

@@ -288,7 +288,7 @@ class dbutils:
             ...
 
         @staticmethod
-        def getArgument(name: str, defaultValue: typing.Optional[str] = None) -> str:
+        def getArgument(name: str, defaultValue: typing.Optional[str] = None) -> typing.Optional[str]:
             """Returns the current value of a widget with give name.
             :param name: Name of the argument to be accessed
             :param defaultValue: (Deprecated) default value

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/service/_internal.py

@@ -3,7 +3,7 @@ from typing import Callable, Dict, Generic, Optional, Type, TypeVar
 
 
 def _from_dict(d: Dict[str, any], field: str, cls: Type) -> any:
-    if field not in d or not d[field]:
+    if field not in d or d[field] is None:
         return None
     return getattr(cls, 'from_dict')(d[field])
 

{databricks-sdk-0.19.1 → databricks-sdk-0.21.0}/databricks/sdk/service/billing.py

@@ -315,6 +315,20 @@ class CreateLogDeliveryConfigurationParams:
                    workspace_ids_filter=d.get('workspace_ids_filter', None))
 
 
+@dataclass
+class DeleteResponse:
+
+    def as_dict(self) -> dict:
+        """Serializes the DeleteResponse into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> DeleteResponse:
+        """Deserializes the DeleteResponse from a dictionary."""
+        return cls()
+
+
 class DeliveryStatus(Enum):
     """The status string for log delivery. Possible values are: * `CREATED`: There were no log delivery
     attempts since the config was created. * `SUCCEEDED`: The latest attempt of log delivery has
@@ -335,6 +349,17 @@ class DeliveryStatus(Enum):
 class DownloadResponse:
     contents: Optional[BinaryIO] = None
 
+    def as_dict(self) -> dict:
+        """Serializes the DownloadResponse into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.contents: body['contents'] = self.contents
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> DownloadResponse:
+        """Deserializes the DownloadResponse from a dictionary."""
+        return cls(contents=d.get('contents', None))
+
 
 class LogDeliveryConfigStatus(Enum):
     """Status of log delivery configuration. Set to `ENABLED` (enabled) or `DISABLED` (disabled).
@@ -546,6 +571,20 @@ class OutputFormat(Enum):
     JSON = 'JSON'
 
 
+@dataclass
+class PatchStatusResponse:
+
+    def as_dict(self) -> dict:
+        """Serializes the PatchStatusResponse into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> PatchStatusResponse:
+        """Deserializes the PatchStatusResponse from a dictionary."""
+        return cls()
+
+
 @dataclass
 class UpdateLogDeliveryConfigurationStatusRequest:
     status: LogDeliveryConfigStatus
@@ -572,6 +611,20 @@ class UpdateLogDeliveryConfigurationStatusRequest:
                    status=_enum(d, 'status', LogDeliveryConfigStatus))
 
 
+@dataclass
+class UpdateResponse:
+
+    def as_dict(self) -> dict:
+        """Serializes the UpdateResponse into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> UpdateResponse:
+        """Deserializes the UpdateResponse from a dictionary."""
+        return cls()
+
+
 @dataclass
 class WrappedBudget:
     budget: Budget
@@ -704,12 +757,13 @@ class BillableUsageAPI:
         if personal_data is not None: query['personal_data'] = personal_data
         if start_month is not None: query['start_month'] = start_month
         headers = {'Accept': 'text/plain', }
+
         res = self._api.do('GET',
                            f'/api/2.0/accounts/{self._api.account_id}/usage/download',
                            query=query,
                            headers=headers,
                            raw=True)
-        return DownloadResponse(contents=res)
+        return DownloadResponse.from_dict(res)
 
 
 class BudgetsAPI:
@@ -732,6 +786,7 @@ class BudgetsAPI:
         body = {}
         if budget is not None: body['budget'] = budget.as_dict()
         headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
+
         res = self._api.do('POST',
                            f'/api/2.0/accounts/{self._api.account_id}/budget',
                            body=body,
@@ -750,6 +805,7 @@ class BudgetsAPI:
         """
 
         headers = {'Accept': 'application/json', }
+
         self._api.do('DELETE',
                      f'/api/2.0/accounts/{self._api.account_id}/budget/{budget_id}',
                      headers=headers)
@@ -767,6 +823,7 @@ class BudgetsAPI:
         """
 
         headers = {'Accept': 'application/json', }
+
         res = self._api.do('GET',
                            f'/api/2.0/accounts/{self._api.account_id}/budget/{budget_id}',
                            headers=headers)
@@ -782,6 +839,7 @@ class BudgetsAPI:
         """
 
         headers = {'Accept': 'application/json', }
+
         json = self._api.do('GET', f'/api/2.0/accounts/{self._api.account_id}/budget', headers=headers)
         parsed = BudgetList.from_dict(json).budgets
         return parsed if parsed is not None else []
@@ -801,6 +859,7 @@ class BudgetsAPI:
         body = {}
         if budget is not None: body['budget'] = budget.as_dict()
         headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
+
         self._api.do('PATCH',
                      f'/api/2.0/accounts/{self._api.account_id}/budget/{budget_id}',
                      body=body,
@@ -894,6 +953,7 @@ class LogDeliveryAPI:
         if log_delivery_configuration is not None:
             body['log_delivery_configuration'] = log_delivery_configuration.as_dict()
         headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
+
         res = self._api.do('POST',
                            f'/api/2.0/accounts/{self._api.account_id}/log-delivery',
                            body=body,
@@ -912,6 +972,7 @@ class LogDeliveryAPI:
         """
 
         headers = {'Accept': 'application/json', }
+
         res = self._api.do(
             'GET',
             f'/api/2.0/accounts/{self._api.account_id}/log-delivery/{log_delivery_configuration_id}',
@@ -942,6 +1003,7 @@ class LogDeliveryAPI:
         if status is not None: query['status'] = status.value
         if storage_configuration_id is not None: query['storage_configuration_id'] = storage_configuration_id
         headers = {'Accept': 'application/json', }
+
         json = self._api.do('GET',
                             f'/api/2.0/accounts/{self._api.account_id}/log-delivery',
                             query=query,
@@ -970,6 +1032,7 @@ class LogDeliveryAPI:
         body = {}
         if status is not None: body['status'] = status.value
         headers = {'Accept': 'application/json', 'Content-Type': 'application/json', }
+
         self._api.do('PATCH',
                      f'/api/2.0/accounts/{self._api.account_id}/log-delivery/{log_delivery_configuration_id}',
                      body=body,