databricks-sdk 0.44.0__py3-none-any.whl → 0.45.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of databricks-sdk might be problematic. Click here for more details.
- databricks/sdk/__init__.py +123 -115
- databricks/sdk/_base_client.py +112 -88
- databricks/sdk/_property.py +12 -7
- databricks/sdk/_widgets/__init__.py +13 -2
- databricks/sdk/_widgets/default_widgets_utils.py +21 -15
- databricks/sdk/_widgets/ipywidgets_utils.py +47 -24
- databricks/sdk/azure.py +8 -6
- databricks/sdk/casing.py +5 -5
- databricks/sdk/config.py +152 -99
- databricks/sdk/core.py +57 -47
- databricks/sdk/credentials_provider.py +360 -210
- databricks/sdk/data_plane.py +86 -3
- databricks/sdk/dbutils.py +123 -87
- databricks/sdk/environments.py +52 -35
- databricks/sdk/errors/base.py +61 -35
- databricks/sdk/errors/customizer.py +3 -3
- databricks/sdk/errors/deserializer.py +38 -25
- databricks/sdk/errors/details.py +417 -0
- databricks/sdk/errors/mapper.py +1 -1
- databricks/sdk/errors/overrides.py +27 -24
- databricks/sdk/errors/parser.py +26 -14
- databricks/sdk/errors/platform.py +10 -10
- databricks/sdk/errors/private_link.py +24 -24
- databricks/sdk/logger/round_trip_logger.py +28 -20
- databricks/sdk/mixins/compute.py +90 -60
- databricks/sdk/mixins/files.py +815 -145
- databricks/sdk/mixins/jobs.py +201 -20
- databricks/sdk/mixins/open_ai_client.py +26 -20
- databricks/sdk/mixins/workspace.py +45 -34
- databricks/sdk/oauth.py +372 -196
- databricks/sdk/retries.py +14 -12
- databricks/sdk/runtime/__init__.py +34 -17
- databricks/sdk/runtime/dbutils_stub.py +52 -39
- databricks/sdk/service/_internal.py +12 -7
- databricks/sdk/service/apps.py +618 -418
- databricks/sdk/service/billing.py +827 -604
- databricks/sdk/service/catalog.py +6552 -4474
- databricks/sdk/service/cleanrooms.py +550 -388
- databricks/sdk/service/compute.py +5241 -3531
- databricks/sdk/service/dashboards.py +1313 -923
- databricks/sdk/service/files.py +442 -309
- databricks/sdk/service/iam.py +2115 -1483
- databricks/sdk/service/jobs.py +4151 -2588
- databricks/sdk/service/marketplace.py +2210 -1517
- databricks/sdk/service/ml.py +3364 -2255
- databricks/sdk/service/oauth2.py +922 -584
- databricks/sdk/service/pipelines.py +1865 -1203
- databricks/sdk/service/provisioning.py +1435 -1029
- databricks/sdk/service/serving.py +2040 -1278
- databricks/sdk/service/settings.py +2846 -1929
- databricks/sdk/service/sharing.py +2201 -877
- databricks/sdk/service/sql.py +4650 -3103
- databricks/sdk/service/vectorsearch.py +816 -550
- databricks/sdk/service/workspace.py +1330 -906
- databricks/sdk/useragent.py +36 -22
- databricks/sdk/version.py +1 -1
- {databricks_sdk-0.44.0.dist-info → databricks_sdk-0.45.0.dist-info}/METADATA +31 -31
- databricks_sdk-0.45.0.dist-info/RECORD +70 -0
- {databricks_sdk-0.44.0.dist-info → databricks_sdk-0.45.0.dist-info}/WHEEL +1 -1
- databricks_sdk-0.44.0.dist-info/RECORD +0 -69
- {databricks_sdk-0.44.0.dist-info → databricks_sdk-0.45.0.dist-info}/LICENSE +0 -0
- {databricks_sdk-0.44.0.dist-info → databricks_sdk-0.45.0.dist-info}/NOTICE +0 -0
- {databricks_sdk-0.44.0.dist-info → databricks_sdk-0.45.0.dist-info}/top_level.txt +0 -0
databricks/sdk/__init__.py
CHANGED
|
@@ -122,6 +122,7 @@ def _make_dbutils(config: client.Config):
|
|
|
122
122
|
|
|
123
123
|
# We are in runtime, so we can use the runtime dbutils
|
|
124
124
|
from databricks.sdk.runtime import dbutils as runtime_dbutils
|
|
125
|
+
|
|
125
126
|
return runtime_dbutils
|
|
126
127
|
|
|
127
128
|
|
|
@@ -138,58 +139,62 @@ class WorkspaceClient:
|
|
|
138
139
|
The WorkspaceClient is a client for the workspace-level Databricks REST API.
|
|
139
140
|
"""
|
|
140
141
|
|
|
141
|
-
def __init__(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
142
|
+
def __init__(
|
|
143
|
+
self,
|
|
144
|
+
*,
|
|
145
|
+
host: Optional[str] = None,
|
|
146
|
+
account_id: Optional[str] = None,
|
|
147
|
+
username: Optional[str] = None,
|
|
148
|
+
password: Optional[str] = None,
|
|
149
|
+
client_id: Optional[str] = None,
|
|
150
|
+
client_secret: Optional[str] = None,
|
|
151
|
+
token: Optional[str] = None,
|
|
152
|
+
profile: Optional[str] = None,
|
|
153
|
+
config_file: Optional[str] = None,
|
|
154
|
+
azure_workspace_resource_id: Optional[str] = None,
|
|
155
|
+
azure_client_secret: Optional[str] = None,
|
|
156
|
+
azure_client_id: Optional[str] = None,
|
|
157
|
+
azure_tenant_id: Optional[str] = None,
|
|
158
|
+
azure_environment: Optional[str] = None,
|
|
159
|
+
auth_type: Optional[str] = None,
|
|
160
|
+
cluster_id: Optional[str] = None,
|
|
161
|
+
google_credentials: Optional[str] = None,
|
|
162
|
+
google_service_account: Optional[str] = None,
|
|
163
|
+
debug_truncate_bytes: Optional[int] = None,
|
|
164
|
+
debug_headers: Optional[bool] = None,
|
|
165
|
+
product="unknown",
|
|
166
|
+
product_version="0.0.0",
|
|
167
|
+
credentials_strategy: Optional[CredentialsStrategy] = None,
|
|
168
|
+
credentials_provider: Optional[CredentialsStrategy] = None,
|
|
169
|
+
config: Optional[client.Config] = None,
|
|
170
|
+
):
|
|
168
171
|
if not config:
|
|
169
|
-
config = client.Config(
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
172
|
+
config = client.Config(
|
|
173
|
+
host=host,
|
|
174
|
+
account_id=account_id,
|
|
175
|
+
username=username,
|
|
176
|
+
password=password,
|
|
177
|
+
client_id=client_id,
|
|
178
|
+
client_secret=client_secret,
|
|
179
|
+
token=token,
|
|
180
|
+
profile=profile,
|
|
181
|
+
config_file=config_file,
|
|
182
|
+
azure_workspace_resource_id=azure_workspace_resource_id,
|
|
183
|
+
azure_client_secret=azure_client_secret,
|
|
184
|
+
azure_client_id=azure_client_id,
|
|
185
|
+
azure_tenant_id=azure_tenant_id,
|
|
186
|
+
azure_environment=azure_environment,
|
|
187
|
+
auth_type=auth_type,
|
|
188
|
+
cluster_id=cluster_id,
|
|
189
|
+
google_credentials=google_credentials,
|
|
190
|
+
google_service_account=google_service_account,
|
|
191
|
+
credentials_strategy=credentials_strategy,
|
|
192
|
+
credentials_provider=credentials_provider,
|
|
193
|
+
debug_truncate_bytes=debug_truncate_bytes,
|
|
194
|
+
debug_headers=debug_headers,
|
|
195
|
+
product=product,
|
|
196
|
+
product_version=product_version,
|
|
197
|
+
)
|
|
193
198
|
self._config = config.copy()
|
|
194
199
|
self._dbutils = _make_dbutils(self._config)
|
|
195
200
|
self._api_client = client.ApiClient(self._config)
|
|
@@ -212,7 +217,8 @@ class WorkspaceClient:
|
|
|
212
217
|
self._consumer_installations = service.marketplace.ConsumerInstallationsAPI(self._api_client)
|
|
213
218
|
self._consumer_listings = service.marketplace.ConsumerListingsAPI(self._api_client)
|
|
214
219
|
self._consumer_personalization_requests = service.marketplace.ConsumerPersonalizationRequestsAPI(
|
|
215
|
-
self._api_client
|
|
220
|
+
self._api_client
|
|
221
|
+
)
|
|
216
222
|
self._consumer_providers = service.marketplace.ConsumerProvidersAPI(self._api_client)
|
|
217
223
|
self._credentials = service.catalog.CredentialsAPI(self._api_client)
|
|
218
224
|
self._credentials_manager = service.settings.CredentialsManagerAPI(self._api_client)
|
|
@@ -246,8 +252,7 @@ class WorkspaceClient:
|
|
|
246
252
|
self._permission_migration = service.iam.PermissionMigrationAPI(self._api_client)
|
|
247
253
|
self._permissions = service.iam.PermissionsAPI(self._api_client)
|
|
248
254
|
self._pipelines = service.pipelines.PipelinesAPI(self._api_client)
|
|
249
|
-
self._policy_compliance_for_clusters = service.compute.PolicyComplianceForClustersAPI(
|
|
250
|
-
self._api_client)
|
|
255
|
+
self._policy_compliance_for_clusters = service.compute.PolicyComplianceForClustersAPI(self._api_client)
|
|
251
256
|
self._policy_compliance_for_jobs = service.jobs.PolicyComplianceForJobsAPI(self._api_client)
|
|
252
257
|
self._policy_families = service.compute.PolicyFamiliesAPI(self._api_client)
|
|
253
258
|
self._provider_exchange_filters = service.marketplace.ProviderExchangeFiltersAPI(self._api_client)
|
|
@@ -255,9 +260,11 @@ class WorkspaceClient:
|
|
|
255
260
|
self._provider_files = service.marketplace.ProviderFilesAPI(self._api_client)
|
|
256
261
|
self._provider_listings = service.marketplace.ProviderListingsAPI(self._api_client)
|
|
257
262
|
self._provider_personalization_requests = service.marketplace.ProviderPersonalizationRequestsAPI(
|
|
258
|
-
self._api_client
|
|
263
|
+
self._api_client
|
|
264
|
+
)
|
|
259
265
|
self._provider_provider_analytics_dashboards = service.marketplace.ProviderProviderAnalyticsDashboardsAPI(
|
|
260
|
-
self._api_client
|
|
266
|
+
self._api_client
|
|
267
|
+
)
|
|
261
268
|
self._provider_providers = service.marketplace.ProviderProvidersAPI(self._api_client)
|
|
262
269
|
self._providers = service.sharing.ProvidersAPI(self._api_client)
|
|
263
270
|
self._quality_monitors = service.catalog.QualityMonitorsAPI(self._api_client)
|
|
@@ -278,7 +285,8 @@ class WorkspaceClient:
|
|
|
278
285
|
self._service_principals = service.iam.ServicePrincipalsAPI(self._api_client)
|
|
279
286
|
self._serving_endpoints = serving_endpoints
|
|
280
287
|
self._serving_endpoints_data_plane = service.serving.ServingEndpointsDataPlaneAPI(
|
|
281
|
-
self._api_client, serving_endpoints
|
|
288
|
+
self._api_client, serving_endpoints
|
|
289
|
+
)
|
|
282
290
|
self._settings = service.settings.SettingsAPI(self._api_client)
|
|
283
291
|
self._shares = service.sharing.SharesAPI(self._api_client)
|
|
284
292
|
self._statement_execution = service.sql.StatementExecutionAPI(self._api_client)
|
|
@@ -606,8 +614,7 @@ class WorkspaceClient:
|
|
|
606
614
|
return self._provider_personalization_requests
|
|
607
615
|
|
|
608
616
|
@property
|
|
609
|
-
def provider_provider_analytics_dashboards(
|
|
610
|
-
self) -> service.marketplace.ProviderProviderAnalyticsDashboardsAPI:
|
|
617
|
+
def provider_provider_analytics_dashboards(self) -> service.marketplace.ProviderProviderAnalyticsDashboardsAPI:
|
|
611
618
|
"""Manage templated analytics solution for providers."""
|
|
612
619
|
return self._provider_provider_analytics_dashboards
|
|
613
620
|
|
|
@@ -803,9 +810,7 @@ class WorkspaceClient:
|
|
|
803
810
|
|
|
804
811
|
def get_workspace_id(self) -> int:
|
|
805
812
|
"""Get the workspace ID of the workspace that this client is connected to."""
|
|
806
|
-
response = self._api_client.do("GET",
|
|
807
|
-
"/api/2.0/preview/scim/v2/Me",
|
|
808
|
-
response_headers=['X-Databricks-Org-Id'])
|
|
813
|
+
response = self._api_client.do("GET", "/api/2.0/preview/scim/v2/Me", response_headers=["X-Databricks-Org-Id"])
|
|
809
814
|
return int(response["X-Databricks-Org-Id"])
|
|
810
815
|
|
|
811
816
|
def __repr__(self):
|
|
@@ -817,58 +822,62 @@ class AccountClient:
|
|
|
817
822
|
The AccountClient is a client for the account-level Databricks REST API.
|
|
818
823
|
"""
|
|
819
824
|
|
|
820
|
-
def __init__(
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
825
|
+
def __init__(
|
|
826
|
+
self,
|
|
827
|
+
*,
|
|
828
|
+
host: Optional[str] = None,
|
|
829
|
+
account_id: Optional[str] = None,
|
|
830
|
+
username: Optional[str] = None,
|
|
831
|
+
password: Optional[str] = None,
|
|
832
|
+
client_id: Optional[str] = None,
|
|
833
|
+
client_secret: Optional[str] = None,
|
|
834
|
+
token: Optional[str] = None,
|
|
835
|
+
profile: Optional[str] = None,
|
|
836
|
+
config_file: Optional[str] = None,
|
|
837
|
+
azure_workspace_resource_id: Optional[str] = None,
|
|
838
|
+
azure_client_secret: Optional[str] = None,
|
|
839
|
+
azure_client_id: Optional[str] = None,
|
|
840
|
+
azure_tenant_id: Optional[str] = None,
|
|
841
|
+
azure_environment: Optional[str] = None,
|
|
842
|
+
auth_type: Optional[str] = None,
|
|
843
|
+
cluster_id: Optional[str] = None,
|
|
844
|
+
google_credentials: Optional[str] = None,
|
|
845
|
+
google_service_account: Optional[str] = None,
|
|
846
|
+
debug_truncate_bytes: Optional[int] = None,
|
|
847
|
+
debug_headers: Optional[bool] = None,
|
|
848
|
+
product="unknown",
|
|
849
|
+
product_version="0.0.0",
|
|
850
|
+
credentials_strategy: Optional[CredentialsStrategy] = None,
|
|
851
|
+
credentials_provider: Optional[CredentialsStrategy] = None,
|
|
852
|
+
config: Optional[client.Config] = None,
|
|
853
|
+
):
|
|
847
854
|
if not config:
|
|
848
|
-
config = client.Config(
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
855
|
+
config = client.Config(
|
|
856
|
+
host=host,
|
|
857
|
+
account_id=account_id,
|
|
858
|
+
username=username,
|
|
859
|
+
password=password,
|
|
860
|
+
client_id=client_id,
|
|
861
|
+
client_secret=client_secret,
|
|
862
|
+
token=token,
|
|
863
|
+
profile=profile,
|
|
864
|
+
config_file=config_file,
|
|
865
|
+
azure_workspace_resource_id=azure_workspace_resource_id,
|
|
866
|
+
azure_client_secret=azure_client_secret,
|
|
867
|
+
azure_client_id=azure_client_id,
|
|
868
|
+
azure_tenant_id=azure_tenant_id,
|
|
869
|
+
azure_environment=azure_environment,
|
|
870
|
+
auth_type=auth_type,
|
|
871
|
+
cluster_id=cluster_id,
|
|
872
|
+
google_credentials=google_credentials,
|
|
873
|
+
google_service_account=google_service_account,
|
|
874
|
+
credentials_strategy=credentials_strategy,
|
|
875
|
+
credentials_provider=credentials_provider,
|
|
876
|
+
debug_truncate_bytes=debug_truncate_bytes,
|
|
877
|
+
debug_headers=debug_headers,
|
|
878
|
+
product=product,
|
|
879
|
+
product_version=product_version,
|
|
880
|
+
)
|
|
872
881
|
self._config = config.copy()
|
|
873
882
|
self._api_client = client.ApiClient(self._config)
|
|
874
883
|
self._access_control = service.iam.AccountAccessControlAPI(self._api_client)
|
|
@@ -888,8 +897,7 @@ class AccountClient:
|
|
|
888
897
|
self._o_auth_published_apps = service.oauth2.OAuthPublishedAppsAPI(self._api_client)
|
|
889
898
|
self._private_access = service.provisioning.PrivateAccessAPI(self._api_client)
|
|
890
899
|
self._published_app_integration = service.oauth2.PublishedAppIntegrationAPI(self._api_client)
|
|
891
|
-
self._service_principal_federation_policy = service.oauth2.ServicePrincipalFederationPolicyAPI(
|
|
892
|
-
self._api_client)
|
|
900
|
+
self._service_principal_federation_policy = service.oauth2.ServicePrincipalFederationPolicyAPI(self._api_client)
|
|
893
901
|
self._service_principal_secrets = service.oauth2.ServicePrincipalSecretsAPI(self._api_client)
|
|
894
902
|
self._service_principals = service.iam.AccountServicePrincipalsAPI(self._api_client)
|
|
895
903
|
self._settings = service.settings.AccountSettingsAPI(self._api_client)
|
databricks/sdk/_base_client.py
CHANGED
|
@@ -17,7 +17,7 @@ from .errors import DatabricksError, _ErrorCustomizer, _Parser
|
|
|
17
17
|
from .logger import RoundTrip
|
|
18
18
|
from .retries import retried
|
|
19
19
|
|
|
20
|
-
logger = logging.getLogger(
|
|
20
|
+
logger = logging.getLogger("databricks.sdk")
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def _fix_host_if_needed(host: Optional[str]) -> Optional[str]:
|
|
@@ -25,35 +25,37 @@ def _fix_host_if_needed(host: Optional[str]) -> Optional[str]:
|
|
|
25
25
|
return host
|
|
26
26
|
|
|
27
27
|
# Add a default scheme if it's missing
|
|
28
|
-
if
|
|
29
|
-
host =
|
|
28
|
+
if "://" not in host:
|
|
29
|
+
host = "https://" + host
|
|
30
30
|
|
|
31
31
|
o = urllib.parse.urlparse(host)
|
|
32
32
|
# remove trailing slash
|
|
33
|
-
path = o.path.rstrip(
|
|
33
|
+
path = o.path.rstrip("/")
|
|
34
34
|
# remove port if 443
|
|
35
35
|
netloc = o.netloc
|
|
36
36
|
if o.port == 443:
|
|
37
|
-
netloc = netloc.split(
|
|
37
|
+
netloc = netloc.split(":")[0]
|
|
38
38
|
|
|
39
39
|
return urllib.parse.urlunparse((o.scheme, netloc, path, o.params, o.query, o.fragment))
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
class _BaseClient:
|
|
43
43
|
|
|
44
|
-
def __init__(
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
debug_truncate_bytes: Optional[int] = None,
|
|
47
|
+
retry_timeout_seconds: Optional[int] = None,
|
|
48
|
+
user_agent_base: Optional[str] = None,
|
|
49
|
+
header_factory: Optional[Callable[[], dict]] = None,
|
|
50
|
+
max_connection_pools: Optional[int] = None,
|
|
51
|
+
max_connections_per_pool: Optional[int] = None,
|
|
52
|
+
pool_block: Optional[bool] = True,
|
|
53
|
+
http_timeout_seconds: Optional[float] = None,
|
|
54
|
+
extra_error_customizers: Optional[List[_ErrorCustomizer]] = None,
|
|
55
|
+
debug_headers: Optional[bool] = False,
|
|
56
|
+
clock: Optional[Clock] = None,
|
|
57
|
+
streaming_buffer_size: int = 1024 * 1024,
|
|
58
|
+
): # 1MB
|
|
57
59
|
"""
|
|
58
60
|
:param debug_truncate_bytes:
|
|
59
61
|
:param retry_timeout_seconds:
|
|
@@ -87,9 +89,11 @@ class _BaseClient:
|
|
|
87
89
|
# We don't use `max_retries` from HTTPAdapter to align with a more production-ready
|
|
88
90
|
# retry strategy established in the Databricks SDK for Go. See _is_retryable and
|
|
89
91
|
# @retried for more details.
|
|
90
|
-
http_adapter = requests.adapters.HTTPAdapter(
|
|
91
|
-
|
|
92
|
-
|
|
92
|
+
http_adapter = requests.adapters.HTTPAdapter(
|
|
93
|
+
pool_connections=max_connections_per_pool or 20,
|
|
94
|
+
pool_maxsize=max_connection_pools or 20,
|
|
95
|
+
pool_block=pool_block,
|
|
96
|
+
)
|
|
93
97
|
self._session.mount("https://", http_adapter)
|
|
94
98
|
|
|
95
99
|
# Default to 60 seconds
|
|
@@ -110,7 +114,7 @@ class _BaseClient:
|
|
|
110
114
|
# See: https://github.com/databricks/databricks-sdk-py/issues/142
|
|
111
115
|
if query is None:
|
|
112
116
|
return None
|
|
113
|
-
with_fixed_bools = {k: v if type(v) != bool else (
|
|
117
|
+
with_fixed_bools = {k: v if type(v) != bool else ("true" if v else "false") for k, v in query.items()}
|
|
114
118
|
|
|
115
119
|
# Query parameters may be nested, e.g.
|
|
116
120
|
# {'filter_by': {'user_ids': [123, 456]}}
|
|
@@ -140,30 +144,34 @@ class _BaseClient:
|
|
|
140
144
|
return False
|
|
141
145
|
return data.seekable()
|
|
142
146
|
|
|
143
|
-
def do(
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
147
|
+
def do(
|
|
148
|
+
self,
|
|
149
|
+
method: str,
|
|
150
|
+
url: str,
|
|
151
|
+
query: Optional[dict] = None,
|
|
152
|
+
headers: Optional[dict] = None,
|
|
153
|
+
body: Optional[dict] = None,
|
|
154
|
+
raw: bool = False,
|
|
155
|
+
files=None,
|
|
156
|
+
data=None,
|
|
157
|
+
auth: Optional[Callable[[requests.PreparedRequest], requests.PreparedRequest]] = None,
|
|
158
|
+
response_headers: Optional[List[str]] = None,
|
|
159
|
+
) -> Union[dict, list, BinaryIO]:
|
|
154
160
|
if headers is None:
|
|
155
161
|
headers = {}
|
|
156
|
-
headers[
|
|
162
|
+
headers["User-Agent"] = self._user_agent_base
|
|
157
163
|
|
|
158
164
|
# Wrap strings and bytes in a seekable stream so that we can rewind them.
|
|
159
165
|
if isinstance(data, (str, bytes)):
|
|
160
|
-
data = io.BytesIO(data.encode(
|
|
166
|
+
data = io.BytesIO(data.encode("utf-8") if isinstance(data, str) else data)
|
|
161
167
|
|
|
162
168
|
if not data:
|
|
163
169
|
# The request is not a stream.
|
|
164
|
-
call = retried(
|
|
165
|
-
|
|
166
|
-
|
|
170
|
+
call = retried(
|
|
171
|
+
timeout=timedelta(seconds=self._retry_timeout_seconds),
|
|
172
|
+
is_retryable=self._is_retryable,
|
|
173
|
+
clock=self._clock,
|
|
174
|
+
)(self._perform)
|
|
167
175
|
elif self._is_seekable_stream(data):
|
|
168
176
|
# Keep track of the initial position of the stream so that we can rewind to it
|
|
169
177
|
# if we need to retry the request.
|
|
@@ -173,25 +181,29 @@ class _BaseClient:
|
|
|
173
181
|
logger.debug(f"Rewinding input data to offset {initial_data_position} before retry")
|
|
174
182
|
data.seek(initial_data_position)
|
|
175
183
|
|
|
176
|
-
call = retried(
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
call = retried(
|
|
185
|
+
timeout=timedelta(seconds=self._retry_timeout_seconds),
|
|
186
|
+
is_retryable=self._is_retryable,
|
|
187
|
+
clock=self._clock,
|
|
188
|
+
before_retry=rewind,
|
|
189
|
+
)(self._perform)
|
|
180
190
|
else:
|
|
181
191
|
# Do not retry if the stream is not seekable. This is necessary to avoid bugs
|
|
182
192
|
# where the retry doesn't re-read already read data from the stream.
|
|
183
193
|
logger.debug(f"Retry disabled for non-seekable stream: type={type(data)}")
|
|
184
194
|
call = self._perform
|
|
185
195
|
|
|
186
|
-
response = call(
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
196
|
+
response = call(
|
|
197
|
+
method,
|
|
198
|
+
url,
|
|
199
|
+
query=query,
|
|
200
|
+
headers=headers,
|
|
201
|
+
body=body,
|
|
202
|
+
raw=raw,
|
|
203
|
+
files=files,
|
|
204
|
+
data=data,
|
|
205
|
+
auth=auth,
|
|
206
|
+
)
|
|
195
207
|
|
|
196
208
|
resp = dict()
|
|
197
209
|
for header in response_headers if response_headers else []:
|
|
@@ -220,6 +232,7 @@ class _BaseClient:
|
|
|
220
232
|
# and Databricks SDK for Go retries
|
|
221
233
|
# (see https://github.com/databricks/databricks-sdk-go/blob/main/apierr/errors.go)
|
|
222
234
|
from urllib3.exceptions import ProxyError
|
|
235
|
+
|
|
223
236
|
if isinstance(err, ProxyError):
|
|
224
237
|
err = err.original_error
|
|
225
238
|
if isinstance(err, requests.ConnectionError):
|
|
@@ -230,48 +243,55 @@ class _BaseClient:
|
|
|
230
243
|
#
|
|
231
244
|
# return a simple string for debug log readability, as `raise TimeoutError(...) from err`
|
|
232
245
|
# will bubble up the original exception in case we reach max retries.
|
|
233
|
-
return f
|
|
246
|
+
return f"cannot connect"
|
|
234
247
|
if isinstance(err, requests.Timeout):
|
|
235
248
|
# corresponds to `TLS handshake timeout` and `i/o timeout` in Go.
|
|
236
249
|
#
|
|
237
250
|
# return a simple string for debug log readability, as `raise TimeoutError(...) from err`
|
|
238
251
|
# will bubble up the original exception in case we reach max retries.
|
|
239
|
-
return f
|
|
252
|
+
return f"timeout"
|
|
240
253
|
if isinstance(err, DatabricksError):
|
|
241
254
|
message = str(err)
|
|
242
255
|
transient_error_string_matches = [
|
|
243
256
|
"com.databricks.backend.manager.util.UnknownWorkerEnvironmentException",
|
|
244
|
-
"does not have any associated worker environments",
|
|
245
|
-
"
|
|
257
|
+
"does not have any associated worker environments",
|
|
258
|
+
"There is no worker environment with id",
|
|
259
|
+
"Unknown worker environment",
|
|
260
|
+
"ClusterNotReadyException",
|
|
261
|
+
"Unexpected error",
|
|
246
262
|
"Please try again later or try a faster operation.",
|
|
247
263
|
"RPC token bucket limit has been exceeded",
|
|
248
264
|
]
|
|
249
265
|
for substring in transient_error_string_matches:
|
|
250
266
|
if substring not in message:
|
|
251
267
|
continue
|
|
252
|
-
return f
|
|
268
|
+
return f"matched {substring}"
|
|
253
269
|
return None
|
|
254
270
|
|
|
255
|
-
def _perform(
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
271
|
+
def _perform(
|
|
272
|
+
self,
|
|
273
|
+
method: str,
|
|
274
|
+
url: str,
|
|
275
|
+
query: Optional[dict] = None,
|
|
276
|
+
headers: Optional[dict] = None,
|
|
277
|
+
body: Optional[dict] = None,
|
|
278
|
+
raw: bool = False,
|
|
279
|
+
files=None,
|
|
280
|
+
data=None,
|
|
281
|
+
auth: Callable[[requests.PreparedRequest], requests.PreparedRequest] = None,
|
|
282
|
+
):
|
|
283
|
+
response = self._session.request(
|
|
284
|
+
method,
|
|
285
|
+
url,
|
|
286
|
+
params=self._fix_query_string(query),
|
|
287
|
+
json=body,
|
|
288
|
+
headers=headers,
|
|
289
|
+
files=files,
|
|
290
|
+
data=data,
|
|
291
|
+
auth=auth,
|
|
292
|
+
stream=raw,
|
|
293
|
+
timeout=self._http_timeout_seconds,
|
|
294
|
+
)
|
|
275
295
|
self._record_request_log(response, raw=raw or data is not None or files is not None)
|
|
276
296
|
error = self._error_parser.get_api_error(response)
|
|
277
297
|
if error is not None:
|
|
@@ -305,14 +325,14 @@ class _StreamingResponse(BinaryIO):
|
|
|
305
325
|
_closed: bool = False
|
|
306
326
|
|
|
307
327
|
def fileno(self) -> int:
|
|
308
|
-
|
|
328
|
+
return 0
|
|
309
329
|
|
|
310
|
-
def flush(self) -> int:
|
|
311
|
-
|
|
330
|
+
def flush(self) -> int: # type: ignore
|
|
331
|
+
return 0
|
|
312
332
|
|
|
313
333
|
def __init__(self, response: _RawResponse, chunk_size: Union[int, None] = None):
|
|
314
334
|
self._response = response
|
|
315
|
-
self._buffer = b
|
|
335
|
+
self._buffer = b""
|
|
316
336
|
self._content = None
|
|
317
337
|
self._chunk_size = chunk_size
|
|
318
338
|
|
|
@@ -338,14 +358,14 @@ class _StreamingResponse(BinaryIO):
|
|
|
338
358
|
|
|
339
359
|
def read(self, n: int = -1) -> bytes:
|
|
340
360
|
"""
|
|
341
|
-
Read up to n bytes from the response stream. If n is negative, read
|
|
342
|
-
until the end of the stream.
|
|
361
|
+
Read up to n bytes from the response stream. If n is negative, read
|
|
362
|
+
until the end of the stream.
|
|
343
363
|
"""
|
|
344
364
|
|
|
345
365
|
self._open()
|
|
346
366
|
read_everything = n < 0
|
|
347
367
|
remaining_bytes = n
|
|
348
|
-
res = b
|
|
368
|
+
res = b""
|
|
349
369
|
while remaining_bytes > 0 or read_everything:
|
|
350
370
|
if len(self._buffer) == 0:
|
|
351
371
|
try:
|
|
@@ -383,10 +403,10 @@ class _StreamingResponse(BinaryIO):
|
|
|
383
403
|
def writable(self) -> bool:
|
|
384
404
|
return False
|
|
385
405
|
|
|
386
|
-
def write(self, s: Union[bytes, bytearray]) -> int:
|
|
406
|
+
def write(self, s: Union[bytes, bytearray]) -> int: # type: ignore
|
|
387
407
|
raise NotImplementedError()
|
|
388
408
|
|
|
389
|
-
def writelines(self, lines: Iterable[bytes]) -> None:
|
|
409
|
+
def writelines(self, lines: Iterable[bytes]) -> None: # type: ignore
|
|
390
410
|
raise NotImplementedError()
|
|
391
411
|
|
|
392
412
|
def __next__(self) -> bytes:
|
|
@@ -395,8 +415,12 @@ class _StreamingResponse(BinaryIO):
|
|
|
395
415
|
def __iter__(self) -> Iterator[bytes]:
|
|
396
416
|
return self._content
|
|
397
417
|
|
|
398
|
-
def __exit__(
|
|
399
|
-
|
|
418
|
+
def __exit__(
|
|
419
|
+
self,
|
|
420
|
+
t: Union[Type[BaseException], None],
|
|
421
|
+
value: Union[BaseException, None],
|
|
422
|
+
traceback: Union[TracebackType, None],
|
|
423
|
+
) -> None:
|
|
400
424
|
self._content = None
|
|
401
|
-
self._buffer = b
|
|
425
|
+
self._buffer = b""
|
|
402
426
|
self.close()
|