spark-kindling-sdk 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spark_kindling_sdk-0.14.0/PKG-INFO +45 -0
- spark_kindling_sdk-0.14.0/README.md +19 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/__init__.py +5 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/artifact_store.py +546 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/env_config.py +79 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/notebooks.py +441 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/platform_api.py +11 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/platform_databricks.py +1628 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/platform_fabric.py +1772 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/platform_provider.py +408 -0
- spark_kindling_sdk-0.14.0/kindling_sdk/platform_synapse.py +1766 -0
- spark_kindling_sdk-0.14.0/pyproject.toml +41 -0
- spark_kindling_sdk-0.14.0/pyproject.toml.orig +40 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark-kindling-sdk
|
|
3
|
+
Version: 0.14.0
|
|
4
|
+
Summary: Design-time platform SDK for Kindling
|
|
5
|
+
Author: SEP Engineering
|
|
6
|
+
Author-email: SEP Engineering <engineering@sep.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Database
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Dist: requests>=2.31.0
|
|
16
|
+
Requires-Dist: azure-identity>=1.15.0
|
|
17
|
+
Requires-Dist: azure-core>=1.30.0,<1.31.0
|
|
18
|
+
Requires-Dist: azure-storage-file-datalake>=12.14.0
|
|
19
|
+
Requires-Dist: databricks-sdk>=0.20.0
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
|
|
22
|
+
Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
|
|
23
|
+
Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
|
|
24
|
+
Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# kindling-sdk
|
|
28
|
+
|
|
29
|
+
Design-time SDK for Kindling:
|
|
30
|
+
|
|
31
|
+
- Platform API client factory for tests/CI
|
|
32
|
+
- Platform-specific remote operations for Fabric, Synapse, and Databricks
|
|
33
|
+
|
|
34
|
+
This package is intentionally separate from runtime bootstrap/runtime execution concerns.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install spark-kindling-sdk
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
`pip install spark-kindling-cli` installs it too (the CLI depends on it). Pin it to the same
|
|
43
|
+
Kindling release as your other Kindling packages (`spark-kindling-sdk==0.14.0`). Without PyPI
|
|
44
|
+
access, install the `spark_kindling_sdk-<version>-py3-none-any.whl` attached to the matching
|
|
45
|
+
[GitHub Release](https://github.com/sep/spark-kindling-framework/releases).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# kindling-sdk
|
|
2
|
+
|
|
3
|
+
Design-time SDK for Kindling:
|
|
4
|
+
|
|
5
|
+
- Platform API client factory for tests/CI
|
|
6
|
+
- Platform-specific remote operations for Fabric, Synapse, and Databricks
|
|
7
|
+
|
|
8
|
+
This package is intentionally separate from runtime bootstrap/runtime execution concerns.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install spark-kindling-sdk
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`pip install spark-kindling-cli` installs it too (the CLI depends on it). Pin it to the same
|
|
17
|
+
Kindling release as your other Kindling packages (`spark-kindling-sdk==0.14.0`). Without PyPI
|
|
18
|
+
access, install the `spark_kindling_sdk-<version>-py3-none-any.whl` attached to the matching
|
|
19
|
+
[GitHub Release](https://github.com/sep/spark-kindling-framework/releases).
|
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
"""Design-time artifact store abstraction (gh#207).
|
|
2
|
+
|
|
3
|
+
The artifacts contract ({root}/config, {root}/packages, {root}/scripts,
|
|
4
|
+
{root}/data-apps/<app>) is read platform-agnostically at runtime — the
|
|
5
|
+
cluster's storage utils interpret whatever string ``artifacts_storage_path``
|
|
6
|
+
holds. This module gives the design-time side (CLI/SDK uploads from a dev
|
|
7
|
+
machine) the same property: the path string declares the backend, and one
|
|
8
|
+
store interface covers every write the tooling performs.
|
|
9
|
+
|
|
10
|
+
Supported artifacts roots:
|
|
11
|
+
|
|
12
|
+
- ``abfss://container@account.dfs.<suffix>/base`` — Azure Blob/ADLS Gen2,
|
|
13
|
+
authenticated via the shared Azure credential chain (service principal
|
|
14
|
+
env triple, then ``DefaultAzureCredential``). Sovereign-cloud endpoints
|
|
15
|
+
resolve exactly as before via ``AZURE_STORAGE_BLOB_ENDPOINT_SUFFIX`` /
|
|
16
|
+
``AZURE_CLOUD``.
|
|
17
|
+
- ``/Volumes/<catalog>/<schema>/<volume>[/base]`` — Unity Catalog volume,
|
|
18
|
+
written through the Databricks Files API. Auth follows the same chain as
|
|
19
|
+
the SDK's ``DatabricksAPI``: ``DATABRICKS_TOKEN``, then the
|
|
20
|
+
``AZURE_TENANT_ID``/``AZURE_CLIENT_ID``/``AZURE_CLIENT_SECRET`` service
|
|
21
|
+
principal, then Azure CLI login (databricks-sdk natively reads ``ARM_*``,
|
|
22
|
+
not ``AZURE_*``, so the explicit mapping here is deliberate).
|
|
23
|
+
- ``file:///path`` or a local directory — filesystem store for tests and
|
|
24
|
+
local development.
|
|
25
|
+
|
|
26
|
+
``dbfs:/`` roots are rejected deliberately: DBFS root storage is deprecated
|
|
27
|
+
by Databricks; use a Unity Catalog volume instead.
|
|
28
|
+
|
|
29
|
+
All store methods take root-relative paths (``config/settings.yaml``,
|
|
30
|
+
``packages/foo.whl``), matching the artifacts folder contract. Third-party
|
|
31
|
+
imports are lazy with actionable install hints.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
import io
|
|
35
|
+
import os
|
|
36
|
+
import posixpath
|
|
37
|
+
from abc import ABC, abstractmethod
|
|
38
|
+
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
39
|
+
from typing import Any, List, Optional, Tuple
|
|
40
|
+
|
|
41
|
+
from kindling_sdk.platform_provider import (
|
|
42
|
+
azure_abfss_uri,
|
|
43
|
+
azure_cloud_config,
|
|
44
|
+
create_azure_credential,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
_SUPPORTED_ROOT_FORMS = (
|
|
48
|
+
"abfss://container@account.dfs.<endpoint-suffix>/path, "
|
|
49
|
+
"/Volumes/<catalog>/<schema>/<volume>[/path], "
|
|
50
|
+
"file:///path, or an existing local directory"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def parse_abfss_uri(uri: str) -> Tuple[str, str, str]:
|
|
55
|
+
"""Parse an abfss:// URI into (container, account, path).
|
|
56
|
+
|
|
57
|
+
Expected format: abfss://container@account.dfs.core.windows.net/path
|
|
58
|
+
The storage account name is the part before the first '.' in the host.
|
|
59
|
+
|
|
60
|
+
Returns (container, storage_account_name, blob_path_prefix).
|
|
61
|
+
Raises ValueError on malformed URIs.
|
|
62
|
+
"""
|
|
63
|
+
if not uri.startswith("abfss://"):
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Invalid destination URI `{uri}`. "
|
|
66
|
+
"Expected abfss://container@account.dfs.core.windows.net/path"
|
|
67
|
+
)
|
|
68
|
+
rest = uri[len("abfss://") :]
|
|
69
|
+
if "@" not in rest:
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"Invalid abfss URI `{uri}`. Missing '@' separator between container and account."
|
|
72
|
+
)
|
|
73
|
+
container, host_and_path = rest.split("@", 1)
|
|
74
|
+
if not container:
|
|
75
|
+
raise ValueError(f"Invalid abfss URI `{uri}`. Container name is empty.")
|
|
76
|
+
|
|
77
|
+
if "/" in host_and_path:
|
|
78
|
+
host, path = host_and_path.split("/", 1)
|
|
79
|
+
else:
|
|
80
|
+
host = host_and_path
|
|
81
|
+
path = ""
|
|
82
|
+
|
|
83
|
+
if "." not in host:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
f"Invalid abfss URI `{uri}`. Host `{host}` does not contain a domain suffix."
|
|
86
|
+
)
|
|
87
|
+
account_name = host.split(".", 1)[0]
|
|
88
|
+
if not account_name:
|
|
89
|
+
raise ValueError(f"Invalid abfss URI `{uri}`. Storage account name is empty.")
|
|
90
|
+
|
|
91
|
+
return container, account_name, path.lstrip("/")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def resolve_blob_account_url(storage_account: str) -> str:
|
|
95
|
+
"""Resolve a storage account name or URL to a full blob endpoint URL.
|
|
96
|
+
|
|
97
|
+
Accepts:
|
|
98
|
+
- Just the account name: "mystorageacct" -> configured cloud blob endpoint
|
|
99
|
+
- Name with custom domain: "mystorageacct.blob.core.usgovcloudapi.net" -> "https://..."
|
|
100
|
+
- Full URL: "https://mystorageacct.blob.core.usgovcloudapi.net" -> passed through
|
|
101
|
+
"""
|
|
102
|
+
if storage_account.startswith("https://"):
|
|
103
|
+
return storage_account
|
|
104
|
+
if "." in storage_account:
|
|
105
|
+
return f"https://{storage_account}"
|
|
106
|
+
endpoint_suffix = os.getenv("AZURE_STORAGE_BLOB_ENDPOINT_SUFFIX")
|
|
107
|
+
if endpoint_suffix:
|
|
108
|
+
endpoint_suffix = endpoint_suffix.strip().rstrip("/").lstrip(".")
|
|
109
|
+
return f"https://{storage_account}.{endpoint_suffix.lstrip('.')}"
|
|
110
|
+
endpoint_suffix = azure_cloud_config()["storage_suffix"]
|
|
111
|
+
return f"https://{storage_account}.blob.{endpoint_suffix.lstrip('.')}"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def create_blob_service_client(storage_account: str):
|
|
115
|
+
"""Create a BlobServiceClient using the shared Azure credential chain."""
|
|
116
|
+
try:
|
|
117
|
+
from azure.storage.blob import BlobServiceClient
|
|
118
|
+
except ImportError as exc:
|
|
119
|
+
raise ImportError(
|
|
120
|
+
"azure-identity and azure-storage-blob are required for abfss:// "
|
|
121
|
+
"artifact stores.\n"
|
|
122
|
+
"Install with: pip install 'spark-kindling-cli[deploy]' "
|
|
123
|
+
"(or: pip install azure-identity azure-storage-blob)"
|
|
124
|
+
) from exc
|
|
125
|
+
|
|
126
|
+
account_url = resolve_blob_account_url(storage_account)
|
|
127
|
+
credential = create_azure_credential(additionally_allowed_tenants=["*"])
|
|
128
|
+
return BlobServiceClient(account_url=account_url, credential=credential)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def create_databricks_workspace_client(host: Optional[str] = None):
|
|
132
|
+
"""Create a databricks-sdk WorkspaceClient with the SDK's auth chain.
|
|
133
|
+
|
|
134
|
+
Token first, then the AZURE_* service principal triple, then Azure CLI
|
|
135
|
+
login — the same order as DatabricksAPI. Kept as a module function so
|
|
136
|
+
stores, the CLI, and (Phase 2) the platform APIs resolve credentials
|
|
137
|
+
identically.
|
|
138
|
+
"""
|
|
139
|
+
try:
|
|
140
|
+
from databricks.sdk import WorkspaceClient
|
|
141
|
+
except ImportError as exc:
|
|
142
|
+
raise ImportError(
|
|
143
|
+
"databricks-sdk is required for /Volumes artifact stores.\n"
|
|
144
|
+
"Install with: pip install databricks-sdk"
|
|
145
|
+
) from exc
|
|
146
|
+
|
|
147
|
+
resolved_host = (host or os.getenv("DATABRICKS_HOST") or "").strip()
|
|
148
|
+
if not resolved_host:
|
|
149
|
+
raise ValueError(
|
|
150
|
+
"DATABRICKS_HOST is required for /Volumes artifact paths — volume "
|
|
151
|
+
"artifact roots are written through the Databricks Files API. Set "
|
|
152
|
+
"DATABRICKS_HOST plus DATABRICKS_TOKEN, the AZURE_TENANT_ID/"
|
|
153
|
+
"AZURE_CLIENT_ID/AZURE_CLIENT_SECRET service principal, or an "
|
|
154
|
+
"Azure CLI login."
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
token = (os.getenv("DATABRICKS_TOKEN") or "").strip()
|
|
158
|
+
if token:
|
|
159
|
+
return WorkspaceClient(host=resolved_host, token=token)
|
|
160
|
+
|
|
161
|
+
tenant_id = (os.getenv("AZURE_TENANT_ID") or "").strip()
|
|
162
|
+
client_id = (os.getenv("AZURE_CLIENT_ID") or "").strip()
|
|
163
|
+
client_secret = (os.getenv("AZURE_CLIENT_SECRET") or "").strip()
|
|
164
|
+
if tenant_id and client_id and client_secret:
|
|
165
|
+
return WorkspaceClient(
|
|
166
|
+
host=resolved_host,
|
|
167
|
+
azure_tenant_id=tenant_id,
|
|
168
|
+
azure_client_id=client_id,
|
|
169
|
+
azure_client_secret=client_secret,
|
|
170
|
+
auth_type="azure-client-secret",
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
return WorkspaceClient(host=resolved_host, auth_type="azure-cli")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _validate_rel_path(rel_path: str) -> str:
|
|
177
|
+
"""Reject paths that would escape the artifacts root.
|
|
178
|
+
|
|
179
|
+
Store paths are root-relative by contract, but they can originate
|
|
180
|
+
outside the caller (e.g. remote listings during store-to-store copies),
|
|
181
|
+
so absolute forms and ``..`` traversal are rejected rather than joined.
|
|
182
|
+
Both posix and Windows interpretations are checked so a path that is
|
|
183
|
+
merely odd on one platform cannot traverse on the other.
|
|
184
|
+
"""
|
|
185
|
+
posix, windows = PurePosixPath(rel_path), PureWindowsPath(rel_path)
|
|
186
|
+
if posix.is_absolute() or windows.is_absolute() or windows.drive:
|
|
187
|
+
raise ValueError(f"Artifact path `{rel_path}` must be root-relative.")
|
|
188
|
+
if ".." in posix.parts or ".." in windows.parts:
|
|
189
|
+
raise ValueError(f"Artifact path `{rel_path}` must not contain `..`.")
|
|
190
|
+
return rel_path
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class ArtifactStore(ABC):
|
|
194
|
+
"""One artifacts root and the writes/reads the design-time tooling needs.
|
|
195
|
+
|
|
196
|
+
All paths are root-relative and forward-slashed, matching the artifacts
|
|
197
|
+
folder contract (``config/``, ``packages/``, ``scripts/``,
|
|
198
|
+
``data-apps/<app>/``).
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
root: str
|
|
202
|
+
|
|
203
|
+
def describe(self) -> str:
|
|
204
|
+
"""Human-readable destination for CLI display lines."""
|
|
205
|
+
return self.root
|
|
206
|
+
|
|
207
|
+
@abstractmethod
|
|
208
|
+
def upload_file(self, rel_path: str, data: bytes, overwrite: bool = True) -> bool:
|
|
209
|
+
"""Upload bytes to ``rel_path``. Returns False when the file already
|
|
210
|
+
exists and ``overwrite`` is False (skip semantics), True otherwise."""
|
|
211
|
+
|
|
212
|
+
@abstractmethod
|
|
213
|
+
def download_file(self, rel_path: str) -> bytes:
|
|
214
|
+
"""Return the contents of ``rel_path``."""
|
|
215
|
+
|
|
216
|
+
@abstractmethod
|
|
217
|
+
def list_files(self, prefix: str = "") -> List[str]:
|
|
218
|
+
"""Recursively list files under ``prefix``, as root-relative paths."""
|
|
219
|
+
|
|
220
|
+
@abstractmethod
|
|
221
|
+
def exists(self, rel_path: str) -> bool:
|
|
222
|
+
"""True when ``rel_path`` exists as a file."""
|
|
223
|
+
|
|
224
|
+
@abstractmethod
|
|
225
|
+
def delete_prefix(self, prefix: str) -> int:
|
|
226
|
+
"""Recursively delete everything under ``prefix``; returns the number
|
|
227
|
+
of files removed."""
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class LocalArtifactStore(ArtifactStore):
|
|
231
|
+
"""Filesystem-backed store (tests and local development)."""
|
|
232
|
+
|
|
233
|
+
def __init__(self, root: str):
|
|
234
|
+
raw = root[len("file://") :] if root.startswith("file://") else root
|
|
235
|
+
self._root_path = Path(raw).expanduser()
|
|
236
|
+
self.root = str(self._root_path)
|
|
237
|
+
|
|
238
|
+
def _full(self, rel_path: str) -> Path:
|
|
239
|
+
return self._root_path / _validate_rel_path(rel_path)
|
|
240
|
+
|
|
241
|
+
def upload_file(self, rel_path: str, data: bytes, overwrite: bool = True) -> bool:
|
|
242
|
+
target = self._full(rel_path)
|
|
243
|
+
if not overwrite and target.exists():
|
|
244
|
+
return False
|
|
245
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
246
|
+
target.write_bytes(data)
|
|
247
|
+
return True
|
|
248
|
+
|
|
249
|
+
def download_file(self, rel_path: str) -> bytes:
|
|
250
|
+
return self._full(rel_path).read_bytes()
|
|
251
|
+
|
|
252
|
+
def list_files(self, prefix: str = "") -> List[str]:
|
|
253
|
+
base = self._full(prefix) if prefix else self._root_path
|
|
254
|
+
if not base.exists():
|
|
255
|
+
return []
|
|
256
|
+
return sorted(
|
|
257
|
+
str(path.relative_to(self._root_path)).replace(os.sep, "/")
|
|
258
|
+
for path in base.rglob("*")
|
|
259
|
+
if path.is_file()
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
def exists(self, rel_path: str) -> bool:
|
|
263
|
+
return self._full(rel_path).is_file()
|
|
264
|
+
|
|
265
|
+
def delete_prefix(self, prefix: str) -> int:
|
|
266
|
+
import shutil
|
|
267
|
+
|
|
268
|
+
base = self._full(prefix) if prefix else self._root_path
|
|
269
|
+
if not base.exists():
|
|
270
|
+
return 0
|
|
271
|
+
count = sum(1 for path in base.rglob("*") if path.is_file())
|
|
272
|
+
shutil.rmtree(base)
|
|
273
|
+
return count
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class AbfssArtifactStore(ArtifactStore):
|
|
277
|
+
"""Azure Blob/ADLS Gen2 store for abfss:// artifact roots.
|
|
278
|
+
|
|
279
|
+
The blob endpoint is rebuilt from the account name via the configured
|
|
280
|
+
Azure environment (``AZURE_STORAGE_BLOB_ENDPOINT_SUFFIX`` /
|
|
281
|
+
``AZURE_CLOUD``), exactly as the CLI resolved it before; the dfs host in
|
|
282
|
+
the URI only contributes the account name. A pre-built client may be
|
|
283
|
+
injected (Phase 2: the platform APIs pass their own).
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
def __init__(self, uri: str, *, blob_service_client: Any = None):
|
|
287
|
+
self.container, self.account, self.base_path = parse_abfss_uri(uri)
|
|
288
|
+
self.root = uri.rstrip("/")
|
|
289
|
+
self._client = blob_service_client
|
|
290
|
+
|
|
291
|
+
@property
|
|
292
|
+
def client(self) -> Any:
|
|
293
|
+
if self._client is None:
|
|
294
|
+
self._client = create_blob_service_client(self.account)
|
|
295
|
+
return self._client
|
|
296
|
+
|
|
297
|
+
def _blob(self, rel_path: str) -> str:
|
|
298
|
+
_validate_rel_path(rel_path)
|
|
299
|
+
return f"{self.base_path}/{rel_path}" if self.base_path else rel_path
|
|
300
|
+
|
|
301
|
+
def _rel(self, blob_name: str) -> str:
|
|
302
|
+
if self.base_path and blob_name.startswith(f"{self.base_path}/"):
|
|
303
|
+
return blob_name[len(self.base_path) + 1 :]
|
|
304
|
+
return blob_name
|
|
305
|
+
|
|
306
|
+
def upload_file(self, rel_path: str, data: bytes, overwrite: bool = True) -> bool:
|
|
307
|
+
blob_client = self.client.get_blob_client(
|
|
308
|
+
container=self.container, blob=self._blob(rel_path)
|
|
309
|
+
)
|
|
310
|
+
if not overwrite:
|
|
311
|
+
try:
|
|
312
|
+
blob_client.get_blob_properties()
|
|
313
|
+
return False # exists, skip
|
|
314
|
+
except Exception:
|
|
315
|
+
pass # doesn't exist, proceed
|
|
316
|
+
blob_client.upload_blob(data, overwrite=True)
|
|
317
|
+
return True
|
|
318
|
+
|
|
319
|
+
def download_file(self, rel_path: str) -> bytes:
|
|
320
|
+
blob_client = self.client.get_blob_client(
|
|
321
|
+
container=self.container, blob=self._blob(rel_path)
|
|
322
|
+
)
|
|
323
|
+
return blob_client.download_blob().readall()
|
|
324
|
+
|
|
325
|
+
def list_files(self, prefix: str = "") -> List[str]:
|
|
326
|
+
full_prefix = self._blob(prefix) if prefix else self.base_path
|
|
327
|
+
list_prefix = (
|
|
328
|
+
f"{full_prefix}/"
|
|
329
|
+
if full_prefix and not full_prefix.endswith("/")
|
|
330
|
+
else (full_prefix or "")
|
|
331
|
+
)
|
|
332
|
+
container_client = self.client.get_container_client(self.container)
|
|
333
|
+
return sorted(
|
|
334
|
+
self._rel(blob.name)
|
|
335
|
+
for blob in container_client.list_blobs(name_starts_with=list_prefix)
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
def exists(self, rel_path: str) -> bool:
|
|
339
|
+
blob_client = self.client.get_blob_client(
|
|
340
|
+
container=self.container, blob=self._blob(rel_path)
|
|
341
|
+
)
|
|
342
|
+
try:
|
|
343
|
+
blob_client.get_blob_properties()
|
|
344
|
+
return True
|
|
345
|
+
except Exception:
|
|
346
|
+
return False
|
|
347
|
+
|
|
348
|
+
def delete_prefix(self, prefix: str) -> int:
|
|
349
|
+
count = 0
|
|
350
|
+
for rel_path in self.list_files(prefix):
|
|
351
|
+
blob_client = self.client.get_blob_client(
|
|
352
|
+
container=self.container, blob=self._blob(rel_path)
|
|
353
|
+
)
|
|
354
|
+
blob_client.delete_blob()
|
|
355
|
+
count += 1
|
|
356
|
+
return count
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
class VolumesArtifactStore(ArtifactStore):
|
|
360
|
+
"""Unity Catalog volume store, written through the Databricks Files API.
|
|
361
|
+
|
|
362
|
+
Requires Databricks credentials (see create_databricks_workspace_client);
|
|
363
|
+
the workspace client may be injected. Files API directory deletes are
|
|
364
|
+
non-recursive, so delete_prefix walks files first, then removes the
|
|
365
|
+
emptied directories deepest-first.
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
def __init__(self, root: str, *, workspace_client: Any = None):
|
|
369
|
+
segments = [segment for segment in root.strip("/").split("/") if segment]
|
|
370
|
+
if len(segments) < 4 or segments[0] != "Volumes":
|
|
371
|
+
raise ValueError(
|
|
372
|
+
f"Invalid volume artifacts path `{root}`. Expected "
|
|
373
|
+
"/Volumes/<catalog>/<schema>/<volume>[/path]."
|
|
374
|
+
)
|
|
375
|
+
self.root = "/" + "/".join(segments)
|
|
376
|
+
self._workspace_client = workspace_client
|
|
377
|
+
if self._workspace_client is None:
|
|
378
|
+
self._workspace_client = create_databricks_workspace_client()
|
|
379
|
+
from databricks.sdk.errors import NotFound
|
|
380
|
+
|
|
381
|
+
self._not_found = NotFound
|
|
382
|
+
|
|
383
|
+
@property
|
|
384
|
+
def _files(self) -> Any:
|
|
385
|
+
return self._workspace_client.files
|
|
386
|
+
|
|
387
|
+
def _full(self, rel_path: str) -> str:
|
|
388
|
+
_validate_rel_path(rel_path)
|
|
389
|
+
return posixpath.join(self.root, rel_path) if rel_path else self.root
|
|
390
|
+
|
|
391
|
+
def _rel(self, full_path: str) -> str:
|
|
392
|
+
return (
|
|
393
|
+
full_path[len(self.root) + 1 :]
|
|
394
|
+
if full_path.startswith(f"{self.root}/")
|
|
395
|
+
else (full_path)
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
def upload_file(self, rel_path: str, data: bytes, overwrite: bool = True) -> bool:
|
|
399
|
+
full = self._full(rel_path)
|
|
400
|
+
if not overwrite and self.exists(rel_path):
|
|
401
|
+
return False
|
|
402
|
+
try:
|
|
403
|
+
self._files.upload(full, io.BytesIO(data), overwrite=True)
|
|
404
|
+
except self._not_found:
|
|
405
|
+
# Some workspace/API versions do not auto-create parent
|
|
406
|
+
# directories on upload.
|
|
407
|
+
self._files.create_directory(posixpath.dirname(full))
|
|
408
|
+
self._files.upload(full, io.BytesIO(data), overwrite=True)
|
|
409
|
+
return True
|
|
410
|
+
|
|
411
|
+
def download_file(self, rel_path: str) -> bytes:
|
|
412
|
+
response = self._files.download(self._full(rel_path))
|
|
413
|
+
return response.contents.read()
|
|
414
|
+
|
|
415
|
+
def list_files(self, prefix: str = "") -> List[str]:
|
|
416
|
+
base = self._full(prefix)
|
|
417
|
+
results: List[str] = []
|
|
418
|
+
stack = [base]
|
|
419
|
+
while stack:
|
|
420
|
+
directory = stack.pop()
|
|
421
|
+
try:
|
|
422
|
+
entries = list(self._files.list_directory_contents(directory))
|
|
423
|
+
except self._not_found:
|
|
424
|
+
continue
|
|
425
|
+
for entry in entries:
|
|
426
|
+
if entry.is_directory:
|
|
427
|
+
stack.append(entry.path)
|
|
428
|
+
else:
|
|
429
|
+
results.append(self._rel(entry.path))
|
|
430
|
+
return sorted(results)
|
|
431
|
+
|
|
432
|
+
def exists(self, rel_path: str) -> bool:
|
|
433
|
+
try:
|
|
434
|
+
self._files.get_metadata(self._full(rel_path))
|
|
435
|
+
return True
|
|
436
|
+
except self._not_found:
|
|
437
|
+
return False
|
|
438
|
+
|
|
439
|
+
def delete_prefix(self, prefix: str) -> int:
|
|
440
|
+
base = self._full(prefix)
|
|
441
|
+
files: List[str] = []
|
|
442
|
+
directories: List[str] = []
|
|
443
|
+
stack = [base]
|
|
444
|
+
while stack:
|
|
445
|
+
directory = stack.pop()
|
|
446
|
+
try:
|
|
447
|
+
entries = list(self._files.list_directory_contents(directory))
|
|
448
|
+
except self._not_found:
|
|
449
|
+
continue
|
|
450
|
+
directories.append(directory)
|
|
451
|
+
for entry in entries:
|
|
452
|
+
if entry.is_directory:
|
|
453
|
+
stack.append(entry.path)
|
|
454
|
+
else:
|
|
455
|
+
files.append(entry.path)
|
|
456
|
+
for file_path in files:
|
|
457
|
+
self._files.delete(file_path)
|
|
458
|
+
for directory in sorted(directories, key=lambda path: path.count("/"), reverse=True):
|
|
459
|
+
try:
|
|
460
|
+
self._files.delete_directory(directory)
|
|
461
|
+
except Exception:
|
|
462
|
+
pass # base may be the volume root itself, which is not deletable
|
|
463
|
+
return len(files)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def artifact_store_for(path: str, *, workspace_client: Any = None) -> ArtifactStore:
|
|
467
|
+
"""Return the store implementation declared by ``path``'s shape.
|
|
468
|
+
|
|
469
|
+
The path string is the single source of truth — the same
|
|
470
|
+
``artifacts_storage_path`` value the runtime interprets on-cluster.
|
|
471
|
+
"""
|
|
472
|
+
candidate = (path or "").strip()
|
|
473
|
+
if not candidate:
|
|
474
|
+
raise ValueError(f"Artifacts path is empty. Expected {_SUPPORTED_ROOT_FORMS}.")
|
|
475
|
+
if candidate.startswith("abfss://"):
|
|
476
|
+
return AbfssArtifactStore(candidate)
|
|
477
|
+
if candidate == "/Volumes" or candidate.startswith("/Volumes/"):
|
|
478
|
+
return VolumesArtifactStore(candidate, workspace_client=workspace_client)
|
|
479
|
+
if candidate.startswith("dbfs:/"):
|
|
480
|
+
raise ValueError(
|
|
481
|
+
f"Unsupported artifacts path `{candidate}`: DBFS root storage is "
|
|
482
|
+
"deprecated by Databricks — use a Unity Catalog volume "
|
|
483
|
+
"(/Volumes/<catalog>/<schema>/<volume>/...) instead."
|
|
484
|
+
)
|
|
485
|
+
if candidate.startswith("file://"):
|
|
486
|
+
return LocalArtifactStore(candidate)
|
|
487
|
+
if "://" in candidate:
|
|
488
|
+
raise ValueError(
|
|
489
|
+
f"Unsupported artifacts path `{candidate}`. Expected {_SUPPORTED_ROOT_FORMS}."
|
|
490
|
+
)
|
|
491
|
+
local_candidate = Path(candidate).expanduser()
|
|
492
|
+
if not local_candidate.is_absolute() and not local_candidate.is_dir():
|
|
493
|
+
raise ValueError(
|
|
494
|
+
f"Unsupported artifacts path `{candidate}`. Expected {_SUPPORTED_ROOT_FORMS}."
|
|
495
|
+
)
|
|
496
|
+
return LocalArtifactStore(candidate)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def resolve_artifacts_path(
|
|
500
|
+
explicit: Optional[str] = None,
|
|
501
|
+
*,
|
|
502
|
+
storage_account: Optional[str] = None,
|
|
503
|
+
container: Optional[str] = None,
|
|
504
|
+
base_path: Optional[str] = None,
|
|
505
|
+
) -> str:
|
|
506
|
+
"""Resolve the artifacts destination to a single path string.
|
|
507
|
+
|
|
508
|
+
Precedence: explicit path argument, then explicit legacy storage flags,
|
|
509
|
+
then the ``KINDLING_ARTIFACTS_STORAGE_PATH`` env var (the Dynaconf env
|
|
510
|
+
spelling of the runtime's ``artifacts_storage_path`` config key — one
|
|
511
|
+
value configures both sides), then the legacy ``AZURE_*`` env triple
|
|
512
|
+
synthesized to an abfss:// URI. Raises ValueError when nothing is
|
|
513
|
+
configured.
|
|
514
|
+
"""
|
|
515
|
+
if explicit and explicit.strip():
|
|
516
|
+
return explicit.strip()
|
|
517
|
+
|
|
518
|
+
legacy_flags_given = bool(storage_account or container) or base_path is not None
|
|
519
|
+
if legacy_flags_given:
|
|
520
|
+
account = (storage_account or os.getenv("AZURE_STORAGE_ACCOUNT", "")).strip()
|
|
521
|
+
if not account:
|
|
522
|
+
raise ValueError(
|
|
523
|
+
"Storage account is required. Use --storage-account or set "
|
|
524
|
+
"AZURE_STORAGE_ACCOUNT."
|
|
525
|
+
)
|
|
526
|
+
resolved_container = container or os.getenv("AZURE_CONTAINER", "artifacts")
|
|
527
|
+
resolved_base = base_path if base_path is not None else os.getenv("AZURE_BASE_PATH", "")
|
|
528
|
+
return azure_abfss_uri(resolved_container, account, resolved_base)
|
|
529
|
+
|
|
530
|
+
env_path = os.getenv("KINDLING_ARTIFACTS_STORAGE_PATH", "").strip()
|
|
531
|
+
if env_path:
|
|
532
|
+
return env_path
|
|
533
|
+
|
|
534
|
+
account = os.getenv("AZURE_STORAGE_ACCOUNT", "").strip()
|
|
535
|
+
if account:
|
|
536
|
+
return azure_abfss_uri(
|
|
537
|
+
os.getenv("AZURE_CONTAINER", "artifacts"), account, os.getenv("AZURE_BASE_PATH", "")
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
raise ValueError(
|
|
541
|
+
"Artifacts location is not configured. Provide --artifacts-path, set "
|
|
542
|
+
"KINDLING_ARTIFACTS_STORAGE_PATH (e.g. "
|
|
543
|
+
"abfss://artifacts@acct.dfs.core.windows.net/kindling or "
|
|
544
|
+
"/Volumes/<catalog>/<schema>/<volume>/kindling), or set the legacy "
|
|
545
|
+
"AZURE_STORAGE_ACCOUNT (+ optional AZURE_CONTAINER, AZURE_BASE_PATH)."
|
|
546
|
+
)
|