brickhouse 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brickhouse/__init__.py +6 -0
- brickhouse/__main__.py +5 -0
- brickhouse/_version.py +3 -0
- brickhouse/api/__init__.py +1 -0
- brickhouse/api/catalog.py +351 -0
- brickhouse/api/compute.py +282 -0
- brickhouse/api/host_metadata.py +58 -0
- brickhouse/api/jobs.py +651 -0
- brickhouse/api/statements.py +431 -0
- brickhouse/api/warehouses.py +222 -0
- brickhouse/api/workspace_files.py +386 -0
- brickhouse/catalog/__init__.py +51 -0
- brickhouse/catalog/identifiers.py +273 -0
- brickhouse/catalog/repository.py +532 -0
- brickhouse/catalog/service.py +43 -0
- brickhouse/cli.py +145 -0
- brickhouse/compute/__init__.py +63 -0
- brickhouse/compute/models.py +352 -0
- brickhouse/compute/repository.py +380 -0
- brickhouse/compute/service.py +921 -0
- brickhouse/compute/warehouse.py +1089 -0
- brickhouse/diagnostic_server.py +286 -0
- brickhouse/diagnostics.py +89 -0
- brickhouse/engines/__init__.py +23 -0
- brickhouse/engines/sparkless.py +1107 -0
- brickhouse/jobs/__init__.py +65 -0
- brickhouse/jobs/execution_context.py +844 -0
- brickhouse/jobs/models.py +299 -0
- brickhouse/jobs/repository.py +1246 -0
- brickhouse/jobs/service.py +1393 -0
- brickhouse/jobs/terraform.py +233 -0
- brickhouse/notebooks/__init__.py +23 -0
- brickhouse/notebooks/runner.py +1072 -0
- brickhouse/notebooks/source_format.py +341 -0
- brickhouse/runtime/__init__.py +81 -0
- brickhouse/runtime/dbutils_local.py +737 -0
- brickhouse/runtime/notebook_kernel.py +236 -0
- brickhouse/runtime/notebook_worker.py +155 -0
- brickhouse/runtime/outcomes.py +276 -0
- brickhouse/runtime/secrets.py +208 -0
- brickhouse/runtime/worker.py +687 -0
- brickhouse/runtime/worker_auth.py +301 -0
- brickhouse/runtime/worker_delta.py +125 -0
- brickhouse/runtime/worker_entry.py +111 -0
- brickhouse/runtime/worker_imports.py +303 -0
- brickhouse/runtime/worker_services.py +309 -0
- brickhouse/runtime/worker_session.py +833 -0
- brickhouse/sql/__init__.py +44 -0
- brickhouse/sql/cancellation.py +131 -0
- brickhouse/sql/errors.py +13 -0
- brickhouse/sql/parameters.py +310 -0
- brickhouse/sql/planner.py +1098 -0
- brickhouse/sql/render.py +31 -0
- brickhouse/sql/results.py +393 -0
- brickhouse/sql/service.py +936 -0
- brickhouse/sql/sessions.py +131 -0
- brickhouse/sql/statements.py +968 -0
- brickhouse/tables/__init__.py +105 -0
- brickhouse/tables/contracts.py +468 -0
- brickhouse/tables/delta_rs.py +1603 -0
- brickhouse/tables/journal.py +421 -0
- brickhouse/tables/staging.py +190 -0
- brickhouse/workspace/__init__.py +73 -0
- brickhouse/workspace/config.py +124 -0
- brickhouse/workspace/context.py +196 -0
- brickhouse/workspace/files.py +322 -0
- brickhouse/workspace/lifecycle.py +2142 -0
- brickhouse/workspace/migrations.py +137 -0
- brickhouse/workspace/ownership.py +152 -0
- brickhouse/workspace/snapshot.py +636 -0
- brickhouse-0.1.0.dist-info/METADATA +325 -0
- brickhouse-0.1.0.dist-info/RECORD +86 -0
- brickhouse-0.1.0.dist-info/WHEEL +4 -0
- brickhouse-0.1.0.dist-info/entry_points.txt +2 -0
- brickhouse-0.1.0.dist-info/licenses/LICENSE +202 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/deltalake-1.2.1-LICENSE.txt +179 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/ipykernel-7.4.0-LICENSE.txt +30 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/jupyter-client-8.10.0-LICENSE.txt +31 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/nbclient-0.11.0-LICENSE.txt +30 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/nbformat-5.11.1-LICENSE.txt +31 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/pyarrow-23.0.1-LICENSE.txt +2331 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/pyarrow-23.0.1-NOTICE.txt +81 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/sparkless-4.15.1-LICENSE.txt +21 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/sparkless-4.15.2-LICENSE.txt +21 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES/sqlglot-30.24.0-LICENSE.txt +21 -0
- brickhouse-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +94 -0
brickhouse/__init__.py
ADDED
brickhouse/__main__.py
ADDED
brickhouse/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Local HTTP transports for Databricks-compatible API surfaces."""
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
"""Small Unity Catalog REST profile backed by local catalog services."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import binascii
|
|
7
|
+
import json
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from http import HTTPStatus
|
|
10
|
+
from typing import TypeVar
|
|
11
|
+
from urllib.parse import parse_qs, unquote, urlsplit
|
|
12
|
+
|
|
13
|
+
from brickhouse.catalog.repository import (
|
|
14
|
+
CatalogRecord,
|
|
15
|
+
CatalogRepositoryError,
|
|
16
|
+
NamespaceAlreadyExistsError,
|
|
17
|
+
NamespaceInUseError,
|
|
18
|
+
NamespaceNotFoundError,
|
|
19
|
+
ProtectedNamespaceError,
|
|
20
|
+
SchemaRecord,
|
|
21
|
+
)
|
|
22
|
+
from brickhouse.catalog.service import CatalogService
|
|
23
|
+
from brickhouse.workspace.context import WorkspaceContext
|
|
24
|
+
|
|
25
|
+
_PREFIX = "/api/2.1/unity-catalog/"
|
|
26
|
+
_DEFAULT_PAGE_SIZE = 1000
|
|
27
|
+
_PageItem = TypeVar("_PageItem")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class ApiResponse:
|
|
32
|
+
status: HTTPStatus
|
|
33
|
+
payload: object
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True, slots=True)
|
|
37
|
+
class ApiBinaryResponse:
|
|
38
|
+
"""A local file download whose body must not be JSON encoded."""
|
|
39
|
+
|
|
40
|
+
status: HTTPStatus
|
|
41
|
+
body: bytes
|
|
42
|
+
content_type: str = "application/octet-stream"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class CatalogHttpApi:
|
|
46
|
+
"""Transport mapping for the SDK catalog/schema methods in B007.3."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, context: WorkspaceContext) -> None:
|
|
49
|
+
self._service = CatalogService(context)
|
|
50
|
+
|
|
51
|
+
def dispatch(
|
|
52
|
+
self,
|
|
53
|
+
method: str,
|
|
54
|
+
target: str,
|
|
55
|
+
body: dict[str, object] | None = None,
|
|
56
|
+
) -> ApiResponse | None:
|
|
57
|
+
parsed = urlsplit(target)
|
|
58
|
+
path = parsed.path
|
|
59
|
+
if not path.startswith(_PREFIX):
|
|
60
|
+
return None
|
|
61
|
+
segments = [unquote(item) for item in path.strip("/").split("/")]
|
|
62
|
+
query = parse_qs(parsed.query, keep_blank_values=True)
|
|
63
|
+
try:
|
|
64
|
+
if segments == ["api", "2.1", "unity-catalog", "catalogs"]:
|
|
65
|
+
return self._catalog_collection(method, query, body)
|
|
66
|
+
if segments == ["api", "2.1", "unity-catalog", "schemas"]:
|
|
67
|
+
return self._schema_collection(method, query, body)
|
|
68
|
+
if len(segments) == 5 and segments[:4] == [
|
|
69
|
+
"api",
|
|
70
|
+
"2.1",
|
|
71
|
+
"unity-catalog",
|
|
72
|
+
"catalogs",
|
|
73
|
+
]:
|
|
74
|
+
return self._catalog_member(method, segments[4], query)
|
|
75
|
+
if len(segments) == 5 and segments[:4] == [
|
|
76
|
+
"api",
|
|
77
|
+
"2.1",
|
|
78
|
+
"unity-catalog",
|
|
79
|
+
"schemas",
|
|
80
|
+
]:
|
|
81
|
+
return self._schema_member(method, segments[4], query)
|
|
82
|
+
return _error(HTTPStatus.NOT_FOUND, "RESOURCE_DOES_NOT_EXIST", "API route not found")
|
|
83
|
+
except NamespaceAlreadyExistsError as error:
|
|
84
|
+
return _error(HTTPStatus.CONFLICT, "ALREADY_EXISTS", str(error))
|
|
85
|
+
except NamespaceNotFoundError as error:
|
|
86
|
+
return _error(HTTPStatus.NOT_FOUND, "RESOURCE_DOES_NOT_EXIST", str(error))
|
|
87
|
+
except NamespaceInUseError as error:
|
|
88
|
+
return _error(HTTPStatus.CONFLICT, "RESOURCE_IN_USE", str(error))
|
|
89
|
+
except ProtectedNamespaceError as error:
|
|
90
|
+
return _error(HTTPStatus.BAD_REQUEST, "INVALID_PARAMETER_VALUE", str(error))
|
|
91
|
+
except CatalogRepositoryError as error:
|
|
92
|
+
return _error(HTTPStatus.BAD_REQUEST, "INVALID_PARAMETER_VALUE", str(error))
|
|
93
|
+
except _InputError as error:
|
|
94
|
+
return _error(HTTPStatus.BAD_REQUEST, "INVALID_PARAMETER_VALUE", str(error))
|
|
95
|
+
|
|
96
|
+
def _catalog_collection(
|
|
97
|
+
self,
|
|
98
|
+
method: str,
|
|
99
|
+
query: dict[str, list[str]],
|
|
100
|
+
body: dict[str, object] | None,
|
|
101
|
+
) -> ApiResponse:
|
|
102
|
+
if method == "POST":
|
|
103
|
+
_validate_query_fields(query, set())
|
|
104
|
+
data = _require_body(body, {"name", "comment"})
|
|
105
|
+
name = _required_string(data, "name")
|
|
106
|
+
comment = _optional_string(data, "comment")
|
|
107
|
+
record = self._service.create_catalog(name, comment=comment)
|
|
108
|
+
return _ok(_catalog_payload(record))
|
|
109
|
+
if method == "GET":
|
|
110
|
+
_validate_query_fields(
|
|
111
|
+
query, {"include_browse", "include_unbound", "max_results", "page_token"}
|
|
112
|
+
)
|
|
113
|
+
_optional_bool_query(query, "include_browse")
|
|
114
|
+
_optional_bool_query(query, "include_unbound")
|
|
115
|
+
page_size = _page_size(query)
|
|
116
|
+
records = self._service.list_catalogs()
|
|
117
|
+
page, next_token = _page(records, "catalogs", query, page_size)
|
|
118
|
+
payload: dict[str, object] = {"catalogs": [_catalog_payload(item) for item in page]}
|
|
119
|
+
if next_token is not None:
|
|
120
|
+
payload["next_page_token"] = next_token
|
|
121
|
+
return _ok(payload)
|
|
122
|
+
return _method_not_allowed("GET, POST")
|
|
123
|
+
|
|
124
|
+
def _catalog_member(
|
|
125
|
+
self,
|
|
126
|
+
method: str,
|
|
127
|
+
name: str,
|
|
128
|
+
query: dict[str, list[str]],
|
|
129
|
+
) -> ApiResponse:
|
|
130
|
+
if method == "GET":
|
|
131
|
+
_validate_query_fields(query, {"include_browse"})
|
|
132
|
+
_optional_bool_query(query, "include_browse")
|
|
133
|
+
return _ok(_catalog_payload(self._service.get_catalog(name)))
|
|
134
|
+
if method == "DELETE":
|
|
135
|
+
_validate_query_fields(query, {"force"})
|
|
136
|
+
_optional_bool_query(query, "force")
|
|
137
|
+
self._service.delete_catalog(name)
|
|
138
|
+
return _ok({})
|
|
139
|
+
_validate_query_fields(query, set())
|
|
140
|
+
return _method_not_allowed("GET, DELETE")
|
|
141
|
+
|
|
142
|
+
def _schema_collection(
|
|
143
|
+
self,
|
|
144
|
+
method: str,
|
|
145
|
+
query: dict[str, list[str]],
|
|
146
|
+
body: dict[str, object] | None,
|
|
147
|
+
) -> ApiResponse:
|
|
148
|
+
if method == "POST":
|
|
149
|
+
_validate_query_fields(query, set())
|
|
150
|
+
data = _require_body(body, {"catalog_name", "name", "comment"})
|
|
151
|
+
catalog_name = _required_string(data, "catalog_name")
|
|
152
|
+
name = _required_string(data, "name")
|
|
153
|
+
comment = _optional_string(data, "comment")
|
|
154
|
+
record = self._service.create_schema(catalog_name, name, comment=comment)
|
|
155
|
+
return _ok(_schema_payload(record))
|
|
156
|
+
if method == "GET":
|
|
157
|
+
_validate_query_fields(
|
|
158
|
+
query, {"catalog_name", "include_browse", "max_results", "page_token"}
|
|
159
|
+
)
|
|
160
|
+
_optional_bool_query(query, "include_browse")
|
|
161
|
+
catalog_name = _required_query_string(query, "catalog_name")
|
|
162
|
+
page_size = _page_size(query)
|
|
163
|
+
records = self._service.list_schemas(catalog_name)
|
|
164
|
+
scope = f"schemas:{catalog_name}"
|
|
165
|
+
page, next_token = _page(records, scope, query, page_size)
|
|
166
|
+
payload: dict[str, object] = {"schemas": [_schema_payload(item) for item in page]}
|
|
167
|
+
if next_token is not None:
|
|
168
|
+
payload["next_page_token"] = next_token
|
|
169
|
+
return _ok(payload)
|
|
170
|
+
return _method_not_allowed("GET, POST")
|
|
171
|
+
|
|
172
|
+
def _schema_member(
|
|
173
|
+
self,
|
|
174
|
+
method: str,
|
|
175
|
+
full_name: str,
|
|
176
|
+
query: dict[str, list[str]],
|
|
177
|
+
) -> ApiResponse:
|
|
178
|
+
parts = full_name.split(".")
|
|
179
|
+
if len(parts) != 2 or not all(parts):
|
|
180
|
+
return _error(
|
|
181
|
+
HTTPStatus.BAD_REQUEST,
|
|
182
|
+
"INVALID_PARAMETER_VALUE",
|
|
183
|
+
"schema full_name must contain catalog and schema names",
|
|
184
|
+
)
|
|
185
|
+
catalog_name, schema_name = parts
|
|
186
|
+
if method == "GET":
|
|
187
|
+
_validate_query_fields(query, {"include_browse"})
|
|
188
|
+
_optional_bool_query(query, "include_browse")
|
|
189
|
+
return _ok(_schema_payload(self._service.get_schema(catalog_name, schema_name)))
|
|
190
|
+
if method == "DELETE":
|
|
191
|
+
_validate_query_fields(query, {"force"})
|
|
192
|
+
_optional_bool_query(query, "force")
|
|
193
|
+
self._service.delete_schema(catalog_name, schema_name)
|
|
194
|
+
return _ok({})
|
|
195
|
+
_validate_query_fields(query, set())
|
|
196
|
+
return _method_not_allowed("GET, DELETE")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _catalog_payload(record: CatalogRecord) -> dict[str, object]:
|
|
200
|
+
return {
|
|
201
|
+
"catalog_type": "MANAGED_CATALOG",
|
|
202
|
+
"comment": record.comment,
|
|
203
|
+
"full_name": record.name,
|
|
204
|
+
"name": record.name,
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _schema_payload(record: SchemaRecord) -> dict[str, object]:
|
|
209
|
+
return {
|
|
210
|
+
"catalog_name": record.catalog_name,
|
|
211
|
+
"comment": record.comment,
|
|
212
|
+
"full_name": f"{record.catalog_name}.{record.name}",
|
|
213
|
+
"name": record.name,
|
|
214
|
+
"schema_id": record.schema_id,
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _require_body(
|
|
219
|
+
body: dict[str, object] | None,
|
|
220
|
+
allowed: set[str],
|
|
221
|
+
) -> dict[str, object]:
|
|
222
|
+
if body is None:
|
|
223
|
+
raise _InputError("JSON object body is required")
|
|
224
|
+
unsupported = set(body) - allowed
|
|
225
|
+
if unsupported:
|
|
226
|
+
raise _InputError(f"unsupported request field(s): {', '.join(sorted(unsupported))}")
|
|
227
|
+
return body
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _required_string(body: dict[str, object], key: str) -> str:
|
|
231
|
+
value = body.get(key)
|
|
232
|
+
if not isinstance(value, str) or not value:
|
|
233
|
+
raise _InputError(f"{key} must be a non-empty string")
|
|
234
|
+
return value
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _optional_string(body: dict[str, object], key: str) -> str | None:
|
|
238
|
+
value = body.get(key)
|
|
239
|
+
if value is None:
|
|
240
|
+
return None
|
|
241
|
+
if not isinstance(value, str):
|
|
242
|
+
raise _InputError(f"{key} must be a string or null")
|
|
243
|
+
return value
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _required_query_string(query: dict[str, list[str]], key: str) -> str:
|
|
247
|
+
value = _single_query_value(query, key)
|
|
248
|
+
if not value:
|
|
249
|
+
raise _InputError(f"query parameter {key} is required")
|
|
250
|
+
return value
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _optional_bool_query(query: dict[str, list[str]], key: str) -> bool | None:
|
|
254
|
+
value = _single_query_value(query, key)
|
|
255
|
+
if value is None:
|
|
256
|
+
return None
|
|
257
|
+
if value.lower() not in {"true", "false"}:
|
|
258
|
+
raise _InputError(f"query parameter {key} must be true or false")
|
|
259
|
+
return value.lower() == "true"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _validate_query_fields(query: dict[str, list[str]], allowed: set[str]) -> None:
|
|
263
|
+
unsupported = set(query) - allowed
|
|
264
|
+
if unsupported:
|
|
265
|
+
raise _InputError(f"unsupported query parameter(s): {', '.join(sorted(unsupported))}")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _single_query_value(query: dict[str, list[str]], key: str) -> str | None:
|
|
269
|
+
values = query.get(key)
|
|
270
|
+
if values is None:
|
|
271
|
+
return None
|
|
272
|
+
if len(values) != 1:
|
|
273
|
+
raise _InputError(f"query parameter {key} must be specified once")
|
|
274
|
+
return values[0]
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _page_size(query: dict[str, list[str]]) -> int | None:
|
|
278
|
+
value = _single_query_value(query, "max_results")
|
|
279
|
+
if value is None:
|
|
280
|
+
return None
|
|
281
|
+
try:
|
|
282
|
+
parsed = int(value)
|
|
283
|
+
except ValueError as error:
|
|
284
|
+
raise _InputError("max_results must be an integer") from error
|
|
285
|
+
if parsed < 0:
|
|
286
|
+
raise _InputError("max_results must be zero or greater")
|
|
287
|
+
return _DEFAULT_PAGE_SIZE if parsed == 0 else parsed
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _page(
|
|
291
|
+
records: list[_PageItem],
|
|
292
|
+
scope: str,
|
|
293
|
+
query: dict[str, list[str]],
|
|
294
|
+
page_size: int | None,
|
|
295
|
+
) -> tuple[list[_PageItem], str | None]:
|
|
296
|
+
token = _single_query_value(query, "page_token")
|
|
297
|
+
offset = _decode_page_token(token, scope) if token else 0
|
|
298
|
+
if offset > len(records):
|
|
299
|
+
raise _InputError("page_token is outside the current result set")
|
|
300
|
+
if page_size is None:
|
|
301
|
+
return records[offset:], None
|
|
302
|
+
page = records[offset : offset + page_size]
|
|
303
|
+
next_offset = offset + len(page)
|
|
304
|
+
next_token = _encode_page_token(scope, next_offset) if next_offset < len(records) else None
|
|
305
|
+
return page, next_token
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _encode_page_token(scope: str, offset: int) -> str:
|
|
309
|
+
raw = json.dumps([scope, offset], separators=(",", ":")).encode("utf-8")
|
|
310
|
+
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _decode_page_token(token: str, expected_scope: str) -> int:
|
|
314
|
+
try:
|
|
315
|
+
raw = base64.urlsafe_b64decode(token + "=" * (-len(token) % 4))
|
|
316
|
+
value = json.loads(raw)
|
|
317
|
+
except (ValueError, binascii.Error, json.JSONDecodeError) as error:
|
|
318
|
+
raise _InputError("page_token is invalid") from error
|
|
319
|
+
if (
|
|
320
|
+
not isinstance(value, list)
|
|
321
|
+
or len(value) != 2
|
|
322
|
+
or value[0] != expected_scope
|
|
323
|
+
or not isinstance(value[1], int)
|
|
324
|
+
or isinstance(value[1], bool)
|
|
325
|
+
or value[1] < 0
|
|
326
|
+
):
|
|
327
|
+
raise _InputError("page_token is invalid for this result set")
|
|
328
|
+
return value[1]
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _error(status: HTTPStatus, code: str, message: str) -> ApiResponse:
|
|
332
|
+
return ApiResponse(status, {"error_code": code, "message": message})
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _ok(payload: dict[str, object]) -> ApiResponse:
|
|
336
|
+
return ApiResponse(HTTPStatus.OK, payload)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _method_not_allowed(allow: str) -> ApiResponse:
|
|
340
|
+
return ApiResponse(
|
|
341
|
+
HTTPStatus.METHOD_NOT_ALLOWED,
|
|
342
|
+
{
|
|
343
|
+
"error_code": "INVALID_PARAMETER_VALUE",
|
|
344
|
+
"message": "method is not supported",
|
|
345
|
+
"allow": allow,
|
|
346
|
+
},
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class _InputError(ValueError):
|
|
351
|
+
pass
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""Pinned Databricks SDK cluster lifecycle routes backed by local metadata."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import binascii
|
|
7
|
+
import json
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from http import HTTPStatus
|
|
10
|
+
from urllib.parse import parse_qs, urlsplit
|
|
11
|
+
|
|
12
|
+
from brickhouse.api.catalog import ApiResponse
|
|
13
|
+
from brickhouse.compute.models import (
|
|
14
|
+
ComputeConflictError,
|
|
15
|
+
ComputeNotFoundError,
|
|
16
|
+
ComputeRecord,
|
|
17
|
+
ComputeStateTransitionError,
|
|
18
|
+
InvalidComputeRequestError,
|
|
19
|
+
UnsupportedComputeEffectError,
|
|
20
|
+
)
|
|
21
|
+
from brickhouse.compute.repository import ComputeRepositoryError
|
|
22
|
+
from brickhouse.compute.service import ComputeService
|
|
23
|
+
from brickhouse.workspace.context import WorkspaceContext
|
|
24
|
+
|
|
25
|
+
_PREFIX = "/api/2.1/clusters/"
|
|
26
|
+
_PAGE_SCOPE = "clusters"
|
|
27
|
+
_MAX_PAGE_SIZE = 1000
|
|
28
|
+
_SIMULATED_STATE_MESSAGE = (
|
|
29
|
+
"Cluster lifecycle metadata is simulated; BrickHouse did not provision cloud compute."
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ComputeHttpApi:
|
|
34
|
+
"""Expose the supported Databricks SDK cluster methods on local metadata."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, context: WorkspaceContext) -> None:
|
|
37
|
+
self._service: ComputeService = context.compute
|
|
38
|
+
|
|
39
|
+
def dispatch(
|
|
40
|
+
self,
|
|
41
|
+
method: str,
|
|
42
|
+
target: str,
|
|
43
|
+
body: dict[str, object] | None = None,
|
|
44
|
+
) -> ApiResponse | None:
|
|
45
|
+
parsed = urlsplit(target)
|
|
46
|
+
if not parsed.path.startswith(_PREFIX):
|
|
47
|
+
return None
|
|
48
|
+
route = parsed.path[len(_PREFIX) :]
|
|
49
|
+
query = parse_qs(parsed.query, keep_blank_values=True)
|
|
50
|
+
try:
|
|
51
|
+
if route == "create":
|
|
52
|
+
if method != "POST":
|
|
53
|
+
return _method_not_allowed("POST")
|
|
54
|
+
_validate_query(query, set())
|
|
55
|
+
record = self._service.create_cluster(_required_body(body))
|
|
56
|
+
return _ok({"cluster_id": record.compute_id})
|
|
57
|
+
if route == "get":
|
|
58
|
+
if method != "GET":
|
|
59
|
+
return _method_not_allowed("GET")
|
|
60
|
+
_validate_query(query, {"cluster_id"})
|
|
61
|
+
compute_id = _required_query(query, "cluster_id")
|
|
62
|
+
return _ok(_cluster_payload(self._service.get_cluster(compute_id)))
|
|
63
|
+
if route == "list":
|
|
64
|
+
if method != "GET":
|
|
65
|
+
return _method_not_allowed("GET")
|
|
66
|
+
_validate_query(query, {"page_size", "page_token", "filter_by.is_pinned"})
|
|
67
|
+
pinned_filter = _single_query(query, "filter_by.is_pinned")
|
|
68
|
+
if pinned_filter is not None and pinned_filter.casefold() not in {"true", "false"}:
|
|
69
|
+
raise _InputError("filter_by.is_pinned must be true or false")
|
|
70
|
+
return _ok(self._list_clusters(query))
|
|
71
|
+
if route == "edit":
|
|
72
|
+
if method != "POST":
|
|
73
|
+
return _method_not_allowed("POST")
|
|
74
|
+
_validate_query(query, set())
|
|
75
|
+
data = _required_body(body)
|
|
76
|
+
compute_id = _take_required_string(data, "cluster_id")
|
|
77
|
+
return _ok(_cluster_payload(self._service.edit_cluster(compute_id, data)))
|
|
78
|
+
if route == "start":
|
|
79
|
+
if method != "POST":
|
|
80
|
+
return _method_not_allowed("POST")
|
|
81
|
+
_validate_query(query, set())
|
|
82
|
+
return _ok(_cluster_payload(self._service.start_cluster(_action_cluster_id(body))))
|
|
83
|
+
if route == "restart":
|
|
84
|
+
if method != "POST":
|
|
85
|
+
return _method_not_allowed("POST")
|
|
86
|
+
_validate_query(query, set())
|
|
87
|
+
data = _required_body(body)
|
|
88
|
+
if set(data) - {"cluster_id"}:
|
|
89
|
+
raise UnsupportedComputeEffectError(
|
|
90
|
+
"restart_user is unsupported because local cluster APIs do not "
|
|
91
|
+
"emulate identity"
|
|
92
|
+
)
|
|
93
|
+
return _ok(
|
|
94
|
+
_cluster_payload(
|
|
95
|
+
self._service.restart_cluster(_take_required_string(data, "cluster_id"))
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
if route == "delete":
|
|
99
|
+
if method != "POST":
|
|
100
|
+
return _method_not_allowed("POST")
|
|
101
|
+
_validate_query(query, set())
|
|
102
|
+
record = self._service.terminate_cluster(_action_cluster_id(body))
|
|
103
|
+
return _ok(_cluster_payload(record))
|
|
104
|
+
if route == "permanent-delete":
|
|
105
|
+
if method != "POST":
|
|
106
|
+
return _method_not_allowed("POST")
|
|
107
|
+
_validate_query(query, set())
|
|
108
|
+
compute_id = _action_cluster_id(body)
|
|
109
|
+
# The Databricks API terminates an active cluster as part of
|
|
110
|
+
# permanent deletion; keep the local lifecycle events aligned.
|
|
111
|
+
self._service.terminate_cluster(compute_id)
|
|
112
|
+
self._service.permanently_delete_cluster(compute_id)
|
|
113
|
+
return _ok({})
|
|
114
|
+
return _error(HTTPStatus.NOT_FOUND, "RESOURCE_DOES_NOT_EXIST", "API route not found")
|
|
115
|
+
except ComputeNotFoundError as error:
|
|
116
|
+
return _error(HTTPStatus.NOT_FOUND, "RESOURCE_DOES_NOT_EXIST", str(error))
|
|
117
|
+
except ComputeConflictError as error:
|
|
118
|
+
return _error(HTTPStatus.CONFLICT, "RESOURCE_CONFLICT", str(error))
|
|
119
|
+
except ComputeStateTransitionError as error:
|
|
120
|
+
return _error(HTTPStatus.CONFLICT, "INVALID_STATE", str(error))
|
|
121
|
+
except (UnsupportedComputeEffectError, InvalidComputeRequestError, _InputError) as error:
|
|
122
|
+
return _error(HTTPStatus.BAD_REQUEST, "INVALID_PARAMETER_VALUE", str(error))
|
|
123
|
+
except ComputeRepositoryError:
|
|
124
|
+
return _error(
|
|
125
|
+
HTTPStatus.INTERNAL_SERVER_ERROR,
|
|
126
|
+
"INTERNAL_ERROR",
|
|
127
|
+
"compute metadata operation failed",
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def _list_clusters(self, query: dict[str, list[str]]) -> dict[str, object]:
|
|
131
|
+
pinned_filter = _single_query(query, "filter_by.is_pinned")
|
|
132
|
+
raw_page_size = _single_query(query, "page_size")
|
|
133
|
+
page_size = _MAX_PAGE_SIZE
|
|
134
|
+
if raw_page_size is not None:
|
|
135
|
+
try:
|
|
136
|
+
page_size = int(raw_page_size)
|
|
137
|
+
except ValueError as error:
|
|
138
|
+
raise _InputError("page_size must be an integer") from error
|
|
139
|
+
if page_size < 1:
|
|
140
|
+
raise _InputError("page_size must be a positive integer")
|
|
141
|
+
page_size = min(page_size, _MAX_PAGE_SIZE)
|
|
142
|
+
token = _single_query(query, "page_token")
|
|
143
|
+
offset = _decode_page_token(token) if token else 0
|
|
144
|
+
if pinned_filter is not None and pinned_filter.casefold() == "true":
|
|
145
|
+
# Cluster pinning is unsupported, so no local cluster can match this
|
|
146
|
+
# provider read used to populate Terraform's is_pinned attribute.
|
|
147
|
+
if offset:
|
|
148
|
+
raise _InputError("page_token is outside the current result set")
|
|
149
|
+
return {"clusters": []}
|
|
150
|
+
total = self._service.count_clusters()
|
|
151
|
+
if offset > total:
|
|
152
|
+
raise _InputError("page_token is outside the current result set")
|
|
153
|
+
page = self._service.list_clusters(page_size=page_size, offset=offset)
|
|
154
|
+
next_offset = offset + len(page)
|
|
155
|
+
result: dict[str, object] = {"clusters": [_cluster_payload(item) for item in page]}
|
|
156
|
+
if next_offset < total:
|
|
157
|
+
result["next_page_token"] = _encode_page_token(next_offset)
|
|
158
|
+
return result
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _cluster_payload(record: ComputeRecord) -> dict[str, object]:
|
|
162
|
+
payload: dict[str, object] = {
|
|
163
|
+
"cluster_id": record.compute_id,
|
|
164
|
+
"cluster_name": record.cluster_name,
|
|
165
|
+
"is_pinned": False,
|
|
166
|
+
"spark_version": record.spark_version,
|
|
167
|
+
"data_security_mode": record.access_mode,
|
|
168
|
+
"runtime_engine": "STANDARD",
|
|
169
|
+
"state": record.state.value,
|
|
170
|
+
"state_message": _SIMULATED_STATE_MESSAGE,
|
|
171
|
+
"start_time": _epoch_millis(record.started_at),
|
|
172
|
+
"terminated_time": _epoch_millis(record.terminated_at),
|
|
173
|
+
"autotermination_minutes": record.autotermination_minutes,
|
|
174
|
+
}
|
|
175
|
+
if record.single_user_name is not None:
|
|
176
|
+
payload["single_user_name"] = record.single_user_name
|
|
177
|
+
settings = record.settings
|
|
178
|
+
for key in (
|
|
179
|
+
"autoscale",
|
|
180
|
+
"custom_tags",
|
|
181
|
+
"driver_node_type_id",
|
|
182
|
+
"enable_elastic_disk",
|
|
183
|
+
"node_type_id",
|
|
184
|
+
"num_workers",
|
|
185
|
+
"spark_conf",
|
|
186
|
+
):
|
|
187
|
+
if key in settings:
|
|
188
|
+
payload[key] = settings[key]
|
|
189
|
+
return payload
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _epoch_millis(value: str | None) -> int | None:
|
|
193
|
+
if value is None:
|
|
194
|
+
return None
|
|
195
|
+
return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _required_body(body: dict[str, object] | None) -> dict[str, object]:
|
|
199
|
+
if not isinstance(body, dict):
|
|
200
|
+
raise _InputError("request body must be a JSON object")
|
|
201
|
+
return dict(body)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _action_cluster_id(body: dict[str, object] | None) -> str:
|
|
205
|
+
data = _required_body(body)
|
|
206
|
+
if set(data) != {"cluster_id"}:
|
|
207
|
+
raise _InputError("request body must contain only cluster_id")
|
|
208
|
+
return _take_required_string(data, "cluster_id")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _take_required_string(data: dict[str, object], key: str) -> str:
|
|
212
|
+
value = data.pop(key, None)
|
|
213
|
+
if not isinstance(value, str) or not value:
|
|
214
|
+
raise _InputError(f"{key} must be a non-empty string")
|
|
215
|
+
return value
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _validate_query(query: dict[str, list[str]], allowed: set[str]) -> None:
|
|
219
|
+
unsupported = set(query) - allowed
|
|
220
|
+
if unsupported:
|
|
221
|
+
raise _InputError(f"unsupported query parameter(s): {', '.join(sorted(unsupported))}")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _single_query(query: dict[str, list[str]], key: str) -> str | None:
|
|
225
|
+
values = query.get(key)
|
|
226
|
+
if values is None:
|
|
227
|
+
return None
|
|
228
|
+
if len(values) != 1:
|
|
229
|
+
raise _InputError(f"query parameter {key} must be specified once")
|
|
230
|
+
return values[0]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _required_query(query: dict[str, list[str]], key: str) -> str:
|
|
234
|
+
value = _single_query(query, key)
|
|
235
|
+
if not value:
|
|
236
|
+
raise _InputError(f"query parameter {key} is required")
|
|
237
|
+
return value
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _encode_page_token(offset: int) -> str:
|
|
241
|
+
return base64.urlsafe_b64encode(json.dumps([_PAGE_SCOPE, offset]).encode()).decode().rstrip("=")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _decode_page_token(token: str) -> int:
|
|
245
|
+
try:
|
|
246
|
+
raw = base64.urlsafe_b64decode(token + "=" * (-len(token) % 4))
|
|
247
|
+
value = json.loads(raw)
|
|
248
|
+
except (ValueError, binascii.Error, json.JSONDecodeError) as error:
|
|
249
|
+
raise _InputError("page_token is invalid") from error
|
|
250
|
+
if (
|
|
251
|
+
not isinstance(value, list)
|
|
252
|
+
or len(value) != 2
|
|
253
|
+
or value[0] != _PAGE_SCOPE
|
|
254
|
+
or not isinstance(value[1], int)
|
|
255
|
+
or isinstance(value[1], bool)
|
|
256
|
+
or value[1] < 0
|
|
257
|
+
):
|
|
258
|
+
raise _InputError("page_token is invalid for this result set")
|
|
259
|
+
return value[1]
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _error(status: HTTPStatus, code: str, message: str) -> ApiResponse:
|
|
263
|
+
return ApiResponse(status, {"error_code": code, "message": message})
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _ok(payload: dict[str, object]) -> ApiResponse:
|
|
267
|
+
return ApiResponse(HTTPStatus.OK, payload)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _method_not_allowed(allow: str) -> ApiResponse:
|
|
271
|
+
return ApiResponse(
|
|
272
|
+
HTTPStatus.METHOD_NOT_ALLOWED,
|
|
273
|
+
{
|
|
274
|
+
"error_code": "INVALID_PARAMETER_VALUE",
|
|
275
|
+
"message": "method is not supported",
|
|
276
|
+
"allow": allow,
|
|
277
|
+
},
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
class _InputError(ValueError):
|
|
282
|
+
pass
|