weaverstack 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weaver/__init__.py +59 -0
- weaver/build_bundle/__init__.py +109 -0
- weaver/build_bundle/aliases.py +325 -0
- weaver/build_bundle/bundle.py +359 -0
- weaver/build_bundle/catalogue_actions.py +275 -0
- weaver/build_bundle/changes.py +186 -0
- weaver/build_bundle/endpoints.py +83 -0
- weaver/build_bundle/executors/__init__.py +69 -0
- weaver/build_bundle/executors/alias.py +202 -0
- weaver/build_bundle/executors/base.py +132 -0
- weaver/build_bundle/executors/folder.py +71 -0
- weaver/build_bundle/executors/load_file.py +205 -0
- weaver/build_bundle/executors/spark_case.py +26 -0
- weaver/build_bundle/executors/spark_schema.py +60 -0
- weaver/build_bundle/executors/spark_sql.py +59 -0
- weaver/build_bundle/executors/spark_sql_batch.py +57 -0
- weaver/build_bundle/executors/spark_table.py +213 -0
- weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
- weaver/build_bundle/executors/tsql.py +81 -0
- weaver/build_bundle/incremental.py +288 -0
- weaver/build_bundle/installer.py +384 -0
- weaver/build_bundle/models.py +288 -0
- weaver/build_bundle/payloads.py +34 -0
- weaver/build_bundle/physical.py +625 -0
- weaver/build_bundle/planner.py +389 -0
- weaver/build_bundle/prune.py +620 -0
- weaver/build_bundle/report.py +108 -0
- weaver/build_bundle/stages.py +196 -0
- weaver/build_bundle/targets.py +272 -0
- weaver/build_bundle/workflow.py +585 -0
- weaver/catalogue/__init__.py +73 -0
- weaver/catalogue/builtin.py +238 -0
- weaver/catalogue/claims.py +121 -0
- weaver/catalogue/projection.py +437 -0
- weaver/catalogue/reader.py +152 -0
- weaver/catalogue/reconcile.py +231 -0
- weaver/catalogue/render.py +410 -0
- weaver/catalogue/state.py +660 -0
- weaver/catalogue/tables.py +648 -0
- weaver/config.py +178 -0
- weaver/declaration/__init__.py +171 -0
- weaver/declaration/columns.py +223 -0
- weaver/declaration/ddl.py +266 -0
- weaver/declaration/dependencies.py +544 -0
- weaver/declaration/graph.py +240 -0
- weaver/declaration/item_dependencies.py +292 -0
- weaver/declaration/load.py +191 -0
- weaver/declaration/metadata.py +1405 -0
- weaver/declaration/model.py +448 -0
- weaver/declaration/references.py +294 -0
- weaver/declaration/repository.py +959 -0
- weaver/declaration/schemas.py +135 -0
- weaver/declaration/source.py +674 -0
- weaver/declaration/spark_load.py +759 -0
- weaver/declaration/sql_shaping.py +591 -0
- weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
- weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
- weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
- weaver/declaration/templates/load/column_metadata.sql +40 -0
- weaver/declaration/templates/load/full_replace_body.sql +21 -0
- weaver/declaration/templates/load/install_load_procedure.sql +27 -0
- weaver/declaration/templates/load/load_procedure.sql +48 -0
- weaver/declaration/templates/load/primary_key_body.sql +113 -0
- weaver/declaration/tsql_ddl.py +468 -0
- weaver/declaration/tsql_load.py +417 -0
- weaver/declaration/warehouse_type_mapping.yml +93 -0
- weaver/diagnostics.py +247 -0
- weaver/errors.py +61 -0
- weaver/etl.py +469 -0
- weaver/fabric/__init__.py +107 -0
- weaver/fabric/auth.py +137 -0
- weaver/fabric/capacity.py +143 -0
- weaver/fabric/client.py +147 -0
- weaver/fabric/environment.py +460 -0
- weaver/fabric/livy.py +478 -0
- weaver/fabric/notebooks.py +201 -0
- weaver/fabric/onelake.py +263 -0
- weaver/fabric/resolution.py +344 -0
- weaver/fabric/resources.py +245 -0
- weaver/fabric/session.py +148 -0
- weaver/fabric/shortcuts.py +120 -0
- weaver/fabric/sql.py +118 -0
- weaver/fabric/store.py +198 -0
- weaver/initialise.py +209 -0
- weaver/lakehouse.py +386 -0
- weaver/load.py +474 -0
- weaver/load_execution.py +483 -0
- weaver/load_plan.py +912 -0
- weaver/load_report.py +330 -0
- weaver/load_resolution.py +386 -0
- weaver/locations.py +164 -0
- weaver/objects.py +392 -0
- weaver/operations.py +757 -0
- weaver/physical_wipe.py +369 -0
- weaver/push.py +76 -0
- weaver/resolution.py +292 -0
- weaver/runtime/__init__.py +30 -0
- weaver/runtime/folder_load.py +402 -0
- weaver/runtime/load_contract.py +245 -0
- weaver/runtime/load_result.py +104 -0
- weaver/runtime/spark_load.py +152 -0
- weaver/runtime/table_load.py +497 -0
- weaver/spark/__init__.py +49 -0
- weaver/spark/catalogue.py +245 -0
- weaver/spark/destination.py +195 -0
- weaver/spark/session.py +84 -0
- weaver/spark/tokens.py +138 -0
- weaver/sql/__init__.py +40 -0
- weaver/sql/authentication.py +38 -0
- weaver/sql/connection.py +90 -0
- weaver/sql/errors.py +25 -0
- weaver/sql/execution.py +123 -0
- weaver/sql/pool.py +174 -0
- weaver/sql/wipe.py +156 -0
- weaver/store.py +209 -0
- weaver/targets.py +257 -0
- weaver/task_logging.py +215 -0
- weaver/unbind.py +74 -0
- weaver/workspaces.py +175 -0
- weaver_cli/__init__.py +12 -0
- weaver_cli/__main__.py +7 -0
- weaver_cli/main.py +626 -0
- weaverstack-0.1.1.dist-info/METADATA +113 -0
- weaverstack-0.1.1.dist-info/RECORD +127 -0
- weaverstack-0.1.1.dist-info/WHEEL +4 -0
- weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
- weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
weaver/operations.py
ADDED
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
"""Source-neutral public build and target-oriented wipe operations.
|
|
2
|
+
|
|
3
|
+
This module is the adaptation boundary between the small notebook-facing API
|
|
4
|
+
and Weaver's typed planning and execution machinery. It deliberately keeps
|
|
5
|
+
all optional platform imports inside the path that needs them so importing
|
|
6
|
+
``weaver`` remains safe without Spark, Fabric credentials, or the desktop CLI.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, replace
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import tempfile
|
|
14
|
+
import uuid
|
|
15
|
+
from typing import Iterable, Mapping, Sequence
|
|
16
|
+
|
|
17
|
+
from .errors import BuildError, CommandError, WeaverError
|
|
18
|
+
from .locations import Location
|
|
19
|
+
from .store import LocalStore, Store
|
|
20
|
+
from .targets import (
|
|
21
|
+
ItemRef,
|
|
22
|
+
WarehouseTarget,
|
|
23
|
+
parse_physical_target,
|
|
24
|
+
physical_item,
|
|
25
|
+
physical_kind,
|
|
26
|
+
)
|
|
27
|
+
from .workspaces import FabricWorkspace, LocalWorkspace, Workspace
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class BuildFailure:
|
|
32
|
+
action_id: str
|
|
33
|
+
error_type: str | None
|
|
34
|
+
message: str | None
|
|
35
|
+
|
|
36
|
+
def to_mapping(self) -> dict:
|
|
37
|
+
return {
|
|
38
|
+
"id": self.action_id,
|
|
39
|
+
"type": self.error_type,
|
|
40
|
+
"message": self.message,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class BuildResult:
|
|
46
|
+
source: str
|
|
47
|
+
items: tuple[str, ...]
|
|
48
|
+
bundle_id: str
|
|
49
|
+
archive: str | None
|
|
50
|
+
status: str
|
|
51
|
+
errors: tuple[BuildFailure, ...] = ()
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def succeeded(self) -> bool:
|
|
55
|
+
return self.status == "succeeded"
|
|
56
|
+
|
|
57
|
+
def to_mapping(self) -> dict:
|
|
58
|
+
return {
|
|
59
|
+
"source": self.source,
|
|
60
|
+
"items": list(self.items),
|
|
61
|
+
"bundle_id": self.bundle_id,
|
|
62
|
+
"archive": self.archive,
|
|
63
|
+
"status": self.status,
|
|
64
|
+
"errors": [error.to_mapping() for error in self.errors],
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class WipeTarget:
|
|
70
|
+
item_type: str
|
|
71
|
+
item: ItemRef
|
|
72
|
+
|
|
73
|
+
@classmethod
|
|
74
|
+
def parse(cls, text: str) -> "WipeTarget":
|
|
75
|
+
target = parse_physical_target(
|
|
76
|
+
text, what="wipe target", error=CommandError
|
|
77
|
+
)
|
|
78
|
+
return cls(item_type=physical_kind(target), item=physical_item(target))
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def physical_name(self) -> str:
|
|
82
|
+
return self.item.name
|
|
83
|
+
|
|
84
|
+
def __str__(self) -> str:
|
|
85
|
+
return f"{self.item_type}/{self.item}"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _unbind_target_names(targets: Iterable[str]) -> tuple[tuple[str, ...], tuple[str, ...]]:
|
|
89
|
+
"""Parse unbind selection through the same typed grammar used by wipe."""
|
|
90
|
+
|
|
91
|
+
parsed = tuple(WipeTarget.parse(target) for target in targets)
|
|
92
|
+
return (
|
|
93
|
+
tuple(target.physical_name for target in parsed if target.item_type == "Lakehouse"),
|
|
94
|
+
tuple(target.physical_name for target in parsed if target.item_type == "Warehouse"),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class WipeReport:
|
|
100
|
+
target: str
|
|
101
|
+
location: Location
|
|
102
|
+
removed: tuple[str, ...]
|
|
103
|
+
dry_run: bool = False
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def count(self) -> int:
|
|
107
|
+
return len(self.removed)
|
|
108
|
+
|
|
109
|
+
def to_mapping(self) -> dict:
|
|
110
|
+
return {
|
|
111
|
+
"target": self.target,
|
|
112
|
+
"location": self.location.value,
|
|
113
|
+
"removed": list(self.removed),
|
|
114
|
+
"dry_run": self.dry_run,
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass(frozen=True)
|
|
119
|
+
class WipeResult:
|
|
120
|
+
workspace: str
|
|
121
|
+
reports: tuple[WipeReport, ...]
|
|
122
|
+
unbound: Mapping | None = None
|
|
123
|
+
dry_run: bool = False
|
|
124
|
+
|
|
125
|
+
@property
|
|
126
|
+
def count(self) -> int:
|
|
127
|
+
return sum(report.count for report in self.reports)
|
|
128
|
+
|
|
129
|
+
def to_mapping(self) -> dict:
|
|
130
|
+
return {
|
|
131
|
+
"workspace": self.workspace,
|
|
132
|
+
"reports": [report.to_mapping() for report in self.reports],
|
|
133
|
+
"unbound": dict(self.unbound) if self.unbound is not None else None,
|
|
134
|
+
"dry_run": self.dry_run,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def build(
|
|
139
|
+
source=None,
|
|
140
|
+
*,
|
|
141
|
+
bind: str | Sequence[str] | None = None,
|
|
142
|
+
workspace: str | Path | Workspace | None = None,
|
|
143
|
+
workspace_config: str | Path | None = None,
|
|
144
|
+
bundle: str | None = None,
|
|
145
|
+
) -> BuildResult:
|
|
146
|
+
"""Build an authored repository using simple notebook-facing values.
|
|
147
|
+
|
|
148
|
+
``workspace=None`` means the current Fabric session. A typed ``Workspace``
|
|
149
|
+
remains accepted as an advanced/testing seam and is what the CLI supplies
|
|
150
|
+
after applying its explicit ``--workspace-type`` flags.
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
resolved_workspace = _operation_workspace(
|
|
154
|
+
workspace=workspace, workspace_config=workspace_config
|
|
155
|
+
)
|
|
156
|
+
resolved_workspace = _with_inferred_control_lakehouse(resolved_workspace)
|
|
157
|
+
if not resolved_workspace.weaver_lakehouse:
|
|
158
|
+
raise CommandError(
|
|
159
|
+
"build needs a Weaver control Lakehouse in workspace configuration "
|
|
160
|
+
"or as the notebook's attached default Lakehouse"
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
selected = _item_bindings(bind, resolved_workspace)
|
|
164
|
+
from .build_bundle.targets import LakehouseBinding, effective_item_bindings
|
|
165
|
+
|
|
166
|
+
bindings = effective_item_bindings(
|
|
167
|
+
selected, weaver_lakehouse=resolved_workspace.weaver_lakehouse
|
|
168
|
+
)
|
|
169
|
+
control = LakehouseBinding(ItemRef(resolved_workspace.weaver_lakehouse))
|
|
170
|
+
source_location, source_store = _repository_source(source, resolved_workspace)
|
|
171
|
+
|
|
172
|
+
# This complete parse and pure request validation is deliberately above all
|
|
173
|
+
# control-plane creation, Spark start, REST item resolution, and Livy work.
|
|
174
|
+
from .build_bundle.workflow import prepare_repository, validate_build_request
|
|
175
|
+
|
|
176
|
+
with prepare_repository(
|
|
177
|
+
source_location, source_store=source_store
|
|
178
|
+
) as prepared:
|
|
179
|
+
validate_build_request(
|
|
180
|
+
prepared.repository, bindings, control_lakehouse=control
|
|
181
|
+
)
|
|
182
|
+
if isinstance(resolved_workspace, LocalWorkspace):
|
|
183
|
+
return _build_local(
|
|
184
|
+
resolved_workspace,
|
|
185
|
+
repository=prepared.repository,
|
|
186
|
+
source_store=prepared.store,
|
|
187
|
+
bindings=bindings,
|
|
188
|
+
control_lakehouse=control,
|
|
189
|
+
bundle_name=bundle,
|
|
190
|
+
source=source_location.value,
|
|
191
|
+
)
|
|
192
|
+
if _inside_fabric_session(resolved_workspace):
|
|
193
|
+
return _build_native_fabric(
|
|
194
|
+
resolved_workspace,
|
|
195
|
+
repository=prepared.repository,
|
|
196
|
+
source_store=prepared.store,
|
|
197
|
+
bindings=bindings,
|
|
198
|
+
control_lakehouse=control,
|
|
199
|
+
bundle_name=bundle,
|
|
200
|
+
source=source_location.value,
|
|
201
|
+
)
|
|
202
|
+
return _build_desktop_fabric(
|
|
203
|
+
resolved_workspace,
|
|
204
|
+
repository=prepared.repository,
|
|
205
|
+
source_store=prepared.store,
|
|
206
|
+
bindings=bindings,
|
|
207
|
+
control_lakehouse=control,
|
|
208
|
+
bundle_name=bundle,
|
|
209
|
+
source=source_location.value,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def wipe(
|
|
214
|
+
targets: str | Iterable[str],
|
|
215
|
+
*,
|
|
216
|
+
workspace: str | Path | Workspace | None = None,
|
|
217
|
+
workspace_config: str | Path | None = None,
|
|
218
|
+
unbind_from: str | None = None,
|
|
219
|
+
dry_run: bool = False,
|
|
220
|
+
) -> WipeResult:
|
|
221
|
+
"""Empty one or more whole Lakehouse or Warehouse items."""
|
|
222
|
+
|
|
223
|
+
values = (targets,) if isinstance(targets, str) else tuple(targets)
|
|
224
|
+
parsed = tuple(WipeTarget.parse(value) for value in values)
|
|
225
|
+
if not parsed:
|
|
226
|
+
raise CommandError("wipe needs at least one target")
|
|
227
|
+
resolved_workspace = _operation_workspace(
|
|
228
|
+
workspace=workspace, workspace_config=workspace_config
|
|
229
|
+
)
|
|
230
|
+
if isinstance(resolved_workspace, LocalWorkspace) and any(
|
|
231
|
+
target.item_type == "Warehouse" for target in parsed
|
|
232
|
+
):
|
|
233
|
+
raise CommandError(
|
|
234
|
+
"Warehouse targets require a Fabric Workspace; the local emulator has no SQL"
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
storage_targets = tuple(t for t in parsed if t.item_type == "Lakehouse")
|
|
238
|
+
store = _operation_store(resolved_workspace) if storage_targets else None
|
|
239
|
+
reports: list[WipeReport] = []
|
|
240
|
+
if not dry_run:
|
|
241
|
+
_drop_local_catalogue(resolved_workspace, storage_targets)
|
|
242
|
+
for target in parsed:
|
|
243
|
+
reports.extend(
|
|
244
|
+
_wipe_one(target, resolved_workspace, store=store, dry_run=dry_run)
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
unbound = None
|
|
248
|
+
control = unbind_from or resolved_workspace.weaver_lakehouse
|
|
249
|
+
whole_lakehouses = {
|
|
250
|
+
target.physical_name
|
|
251
|
+
for target in parsed
|
|
252
|
+
if target.item_type == "Lakehouse"
|
|
253
|
+
}
|
|
254
|
+
if not dry_run and control and control not in whole_lakehouses:
|
|
255
|
+
catalogue_workspace = replace(
|
|
256
|
+
resolved_workspace, weaver_lakehouse=ItemRef.parse(control).name
|
|
257
|
+
)
|
|
258
|
+
unbound = _unbind_physical_targets(catalogue_workspace, parsed)
|
|
259
|
+
|
|
260
|
+
return WipeResult(
|
|
261
|
+
workspace=str(resolved_workspace.workspace),
|
|
262
|
+
reports=tuple(reports),
|
|
263
|
+
unbound=unbound,
|
|
264
|
+
dry_run=dry_run,
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _operation_workspace(*, workspace, workspace_config) -> Workspace:
|
|
269
|
+
if isinstance(workspace, Workspace):
|
|
270
|
+
if workspace_config is not None:
|
|
271
|
+
raise CommandError(
|
|
272
|
+
"workspace_config cannot be combined with an already resolved Workspace"
|
|
273
|
+
)
|
|
274
|
+
return workspace
|
|
275
|
+
if workspace is None and workspace_config is None:
|
|
276
|
+
return _current_fabric_workspace()
|
|
277
|
+
from .config import resolve_workspace
|
|
278
|
+
|
|
279
|
+
return resolve_workspace(workspace=workspace, workspace_config=workspace_config)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _current_fabric_workspace() -> FabricWorkspace:
|
|
283
|
+
try:
|
|
284
|
+
from notebookutils import runtime
|
|
285
|
+
except ImportError as exc:
|
|
286
|
+
raise CommandError(
|
|
287
|
+
"give workspace or workspace_config outside a Fabric notebook"
|
|
288
|
+
) from exc
|
|
289
|
+
context = runtime.context
|
|
290
|
+
if callable(context):
|
|
291
|
+
context = context()
|
|
292
|
+
if not isinstance(context, Mapping):
|
|
293
|
+
raise CommandError("Fabric runtime context is not a mapping")
|
|
294
|
+
name = context.get("currentWorkspaceName")
|
|
295
|
+
if not name:
|
|
296
|
+
raise CommandError("Fabric runtime context carries no current workspace")
|
|
297
|
+
return FabricWorkspace(workspace=str(name))
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _with_inferred_control_lakehouse(workspace: Workspace) -> Workspace:
|
|
301
|
+
if workspace.weaver_lakehouse or not isinstance(workspace, FabricWorkspace):
|
|
302
|
+
return workspace
|
|
303
|
+
if not _inside_fabric_session(workspace):
|
|
304
|
+
return workspace
|
|
305
|
+
from .lakehouse import default_lakehouse
|
|
306
|
+
|
|
307
|
+
spark = _active_spark()
|
|
308
|
+
return replace(workspace, weaver_lakehouse=default_lakehouse(spark).name)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _active_spark():
|
|
312
|
+
try:
|
|
313
|
+
from importlib import import_module
|
|
314
|
+
|
|
315
|
+
SparkSession = import_module("pyspark.sql").SparkSession
|
|
316
|
+
except ImportError as exc:
|
|
317
|
+
raise CommandError("this operation needs an active Spark session") from exc
|
|
318
|
+
spark = SparkSession.getActiveSession()
|
|
319
|
+
if spark is None:
|
|
320
|
+
raise CommandError("this operation needs an active Spark session")
|
|
321
|
+
return spark
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _inside_fabric_session(workspace: FabricWorkspace) -> bool:
|
|
325
|
+
try:
|
|
326
|
+
from notebookutils import runtime
|
|
327
|
+
except ImportError:
|
|
328
|
+
return False
|
|
329
|
+
context = runtime.context
|
|
330
|
+
if callable(context):
|
|
331
|
+
context = context()
|
|
332
|
+
if not isinstance(context, Mapping):
|
|
333
|
+
return False
|
|
334
|
+
return context.get("currentWorkspaceName") == workspace.workspace
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _repository_source(source, workspace: Workspace) -> tuple[Location, Store]:
|
|
338
|
+
if source is None:
|
|
339
|
+
if not isinstance(workspace, FabricWorkspace) or not _inside_fabric_session(workspace):
|
|
340
|
+
source = "."
|
|
341
|
+
else:
|
|
342
|
+
# Fabric exposes built-in Notebook Resources as the notebook's
|
|
343
|
+
# process-local working tree. No OneLake adapter is involved.
|
|
344
|
+
source = Path.cwd()
|
|
345
|
+
location = source if isinstance(source, Location) else Location(str(source))
|
|
346
|
+
if location.value.startswith("abfss://"):
|
|
347
|
+
if not isinstance(workspace, FabricWorkspace) or not _inside_fabric_session(workspace):
|
|
348
|
+
raise CommandError(
|
|
349
|
+
"an abfss repository source can be read only inside a Fabric session"
|
|
350
|
+
)
|
|
351
|
+
from .fabric.store import FabricStore
|
|
352
|
+
|
|
353
|
+
return location, FabricStore()
|
|
354
|
+
return location, LocalStore()
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _item_bindings(bind, workspace: Workspace):
|
|
358
|
+
from .build_bundle.targets import ItemBindings, parse_item_binding
|
|
359
|
+
|
|
360
|
+
if bind is None:
|
|
361
|
+
values = [f"Lakehouse/{name}" for name in workspace.lakehouses]
|
|
362
|
+
values += [f"Warehouse/{name}" for name in workspace.warehouses]
|
|
363
|
+
elif isinstance(bind, str):
|
|
364
|
+
values = [bind]
|
|
365
|
+
else:
|
|
366
|
+
values = list(bind)
|
|
367
|
+
if not values:
|
|
368
|
+
raise BuildError(
|
|
369
|
+
"build needs bind values or configured Lakehouse/Warehouse targets"
|
|
370
|
+
)
|
|
371
|
+
return ItemBindings(
|
|
372
|
+
tuple(parse_item_binding(value, workspace=workspace) for value in values)
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _ensure_control_plane(workspace: Workspace, *, store: Store, spark) -> None:
|
|
377
|
+
from .initialise import initialise_weaver_lakehouse, prepare_weaver_lakehouse
|
|
378
|
+
|
|
379
|
+
resolver = None
|
|
380
|
+
client = None
|
|
381
|
+
if isinstance(workspace, FabricWorkspace) and _inside_fabric_session(workspace):
|
|
382
|
+
from .resolution import resolver_for
|
|
383
|
+
|
|
384
|
+
resolver = resolver_for(workspace)
|
|
385
|
+
factory = getattr(resolver, "_rest_client", None)
|
|
386
|
+
client = factory() if callable(factory) else None
|
|
387
|
+
prepare_weaver_lakehouse(workspace, exists_ok=True, store=store, client=client)
|
|
388
|
+
initialise_weaver_lakehouse(
|
|
389
|
+
weaver_lakehouse=ItemRef(workspace.weaver_lakehouse),
|
|
390
|
+
workspace=workspace,
|
|
391
|
+
store=store,
|
|
392
|
+
spark=spark,
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _archive_location(resolver, bundle_name: str | None):
|
|
397
|
+
if bundle_name is None:
|
|
398
|
+
return None
|
|
399
|
+
from .build_bundle.workflow import timestamped_archive_name
|
|
400
|
+
|
|
401
|
+
name = bundle_name or timestamped_archive_name()
|
|
402
|
+
if not name.endswith(".weaver.zip"):
|
|
403
|
+
name += ".weaver.zip"
|
|
404
|
+
return resolver.build_bundle(name)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _result_from_item_build(source, bindings, result) -> BuildResult:
|
|
408
|
+
report = result.report
|
|
409
|
+
return BuildResult(
|
|
410
|
+
source=source,
|
|
411
|
+
items=tuple(str(binding.item) for binding in bindings.entries),
|
|
412
|
+
bundle_id=result.bundle_id,
|
|
413
|
+
archive=result.archive.value if result.archive else None,
|
|
414
|
+
status=report.status,
|
|
415
|
+
errors=tuple(
|
|
416
|
+
BuildFailure(
|
|
417
|
+
action.action_id, action.error_type, action.error_message
|
|
418
|
+
)
|
|
419
|
+
for action in report.action_results()
|
|
420
|
+
if action.status == "failed"
|
|
421
|
+
),
|
|
422
|
+
)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _build_in_process(
|
|
426
|
+
workspace,
|
|
427
|
+
*,
|
|
428
|
+
spark,
|
|
429
|
+
store,
|
|
430
|
+
repository,
|
|
431
|
+
source_store,
|
|
432
|
+
bindings,
|
|
433
|
+
control_lakehouse,
|
|
434
|
+
bundle_name,
|
|
435
|
+
source,
|
|
436
|
+
) -> BuildResult:
|
|
437
|
+
from .build_bundle import (
|
|
438
|
+
InstallationEnvironment,
|
|
439
|
+
build_item_repository,
|
|
440
|
+
catalogue_items_for_build,
|
|
441
|
+
read_build_state,
|
|
442
|
+
)
|
|
443
|
+
from .catalogue.state import reconcile_catalogue_state
|
|
444
|
+
from .resolution import resolver_for
|
|
445
|
+
|
|
446
|
+
_ensure_control_plane(workspace, store=store, spark=spark)
|
|
447
|
+
resolver = resolver_for(workspace)
|
|
448
|
+
environment = InstallationEnvironment(
|
|
449
|
+
store=store, resolver=resolver, spark=spark, workspace=workspace
|
|
450
|
+
)
|
|
451
|
+
state = read_build_state(
|
|
452
|
+
bindings,
|
|
453
|
+
required_catalogue_items=catalogue_items_for_build(repository, bindings),
|
|
454
|
+
environment=environment,
|
|
455
|
+
)
|
|
456
|
+
result = build_item_repository(
|
|
457
|
+
repository,
|
|
458
|
+
bindings=bindings,
|
|
459
|
+
target_inventories=state.target_inventories,
|
|
460
|
+
reconciliation=reconcile_catalogue_state(
|
|
461
|
+
state.catalogue, inventories=state.target_inventories
|
|
462
|
+
),
|
|
463
|
+
environment=environment,
|
|
464
|
+
source_store=source_store,
|
|
465
|
+
control_lakehouse=control_lakehouse,
|
|
466
|
+
archive=_archive_location(resolver, bundle_name),
|
|
467
|
+
)
|
|
468
|
+
return _result_from_item_build(source, bindings, result)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _build_local(workspace, **kwargs) -> BuildResult:
|
|
472
|
+
warehouse_items = [
|
|
473
|
+
str(binding.item)
|
|
474
|
+
for binding in kwargs["bindings"].entries
|
|
475
|
+
if binding.item.item_type == "Warehouse"
|
|
476
|
+
]
|
|
477
|
+
if warehouse_items:
|
|
478
|
+
raise CommandError(
|
|
479
|
+
"local Workspace builds cannot target Warehouses: "
|
|
480
|
+
+ ", ".join(warehouse_items)
|
|
481
|
+
)
|
|
482
|
+
from .spark import local_delta_session
|
|
483
|
+
|
|
484
|
+
with local_delta_session(workspace) as session:
|
|
485
|
+
return _build_in_process(
|
|
486
|
+
workspace, spark=session, store=LocalStore(), **kwargs
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _build_native_fabric(workspace, **kwargs) -> BuildResult:
|
|
491
|
+
from .resolution import store_for
|
|
492
|
+
|
|
493
|
+
return _build_in_process(
|
|
494
|
+
workspace, spark=_active_spark(), store=store_for(workspace), **kwargs
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _build_desktop_fabric(
|
|
499
|
+
workspace,
|
|
500
|
+
*,
|
|
501
|
+
repository,
|
|
502
|
+
source_store,
|
|
503
|
+
bindings,
|
|
504
|
+
control_lakehouse,
|
|
505
|
+
bundle_name,
|
|
506
|
+
source,
|
|
507
|
+
) -> BuildResult:
|
|
508
|
+
if not workspace.environment:
|
|
509
|
+
raise CommandError(
|
|
510
|
+
"Fabric build requires an Environment in workspace configuration"
|
|
511
|
+
)
|
|
512
|
+
from .build_bundle import (
|
|
513
|
+
BuildState,
|
|
514
|
+
catalogue_items_for_build,
|
|
515
|
+
generate_item_build_bundle,
|
|
516
|
+
persist_bundle_archive,
|
|
517
|
+
)
|
|
518
|
+
from .catalogue.state import reconcile_catalogue_state
|
|
519
|
+
from .fabric import LivySession, OneLakeDfsClient
|
|
520
|
+
from .initialise import prepare_weaver_lakehouse
|
|
521
|
+
from .resolution import resolver_for
|
|
522
|
+
|
|
523
|
+
prepare_weaver_lakehouse(workspace, exists_ok=True)
|
|
524
|
+
resolver = resolver_for(workspace)
|
|
525
|
+
transport_store = OneLakeDfsClient()
|
|
526
|
+
binding_texts = [_binding_text(binding) for binding in bindings.entries]
|
|
527
|
+
required_items = [
|
|
528
|
+
str(item) for item in catalogue_items_for_build(repository, bindings)
|
|
529
|
+
]
|
|
530
|
+
workspace_literal = (
|
|
531
|
+
f"FabricWorkspace(workspace={workspace.workspace!r}, "
|
|
532
|
+
f"weaver_lakehouse={workspace.weaver_lakehouse!r}, "
|
|
533
|
+
f"environment={workspace.environment!r})"
|
|
534
|
+
)
|
|
535
|
+
state_body = (
|
|
536
|
+
"from weaver.workspaces import FabricWorkspace\n"
|
|
537
|
+
"from weaver.declaration.model import WeaverItemId\n"
|
|
538
|
+
"from weaver.build_bundle import (InstallationEnvironment, ItemBindings, "
|
|
539
|
+
"parse_item_binding, read_build_state)\n"
|
|
540
|
+
"from weaver.initialise import initialise_weaver_lakehouse\n"
|
|
541
|
+
"from weaver.targets import ItemRef\n"
|
|
542
|
+
"from weaver.resolution import resolver_for, store_for\n"
|
|
543
|
+
f"workspace = {workspace_literal}\n"
|
|
544
|
+
"store = store_for(workspace)\n"
|
|
545
|
+
"resolver = resolver_for(workspace)\n"
|
|
546
|
+
"initialise_weaver_lakehouse(weaver_lakehouse=ItemRef("
|
|
547
|
+
"workspace.weaver_lakehouse), workspace=workspace, store=store, spark=spark)\n"
|
|
548
|
+
f"bindings = ItemBindings(tuple(parse_item_binding(text) for text in {binding_texts!r}))\n"
|
|
549
|
+
"environment = InstallationEnvironment("
|
|
550
|
+
"store=store, resolver=resolver, spark=spark, workspace=workspace)\n"
|
|
551
|
+
f"items = tuple(WeaverItemId.parse(value) for value in {required_items!r})\n"
|
|
552
|
+
"emit(read_build_state(bindings, required_catalogue_items=items, "
|
|
553
|
+
"environment=environment).to_mapping())\n"
|
|
554
|
+
)
|
|
555
|
+
execution_id = uuid.uuid4().hex
|
|
556
|
+
execution = resolver.cli_execution(execution_id)
|
|
557
|
+
remote_archive = resolver.cli_bundle(execution_id)
|
|
558
|
+
retained_archive = _archive_location(resolver, bundle_name)
|
|
559
|
+
bundle = None
|
|
560
|
+
report = None
|
|
561
|
+
with LivySession.for_workspace(workspace) as session:
|
|
562
|
+
state = BuildState.from_mapping(session.run(state_body).payload)
|
|
563
|
+
reconciliation = reconcile_catalogue_state(
|
|
564
|
+
state.catalogue, inventories=state.target_inventories
|
|
565
|
+
)
|
|
566
|
+
try:
|
|
567
|
+
with tempfile.TemporaryDirectory(prefix="weaver-cli-build-") as temporary:
|
|
568
|
+
root = Path(temporary)
|
|
569
|
+
bundle = generate_item_build_bundle(
|
|
570
|
+
repository,
|
|
571
|
+
bindings=bindings,
|
|
572
|
+
output=Location((root / "bundle").as_posix()),
|
|
573
|
+
store=source_store,
|
|
574
|
+
target_inventories=state.target_inventories,
|
|
575
|
+
catalogue=reconciliation.catalogue,
|
|
576
|
+
stale_claims=reconciliation.stale_claims,
|
|
577
|
+
control_lakehouse=control_lakehouse,
|
|
578
|
+
)
|
|
579
|
+
local_archive = Location((root / "install.weaver.zip").as_posix())
|
|
580
|
+
persist_bundle_archive(bundle, local_archive, store=LocalStore())
|
|
581
|
+
archive_bytes = LocalStore().read(local_archive)
|
|
582
|
+
transport_store.make_directory(resolver.cli_root)
|
|
583
|
+
transport_store.make_directory(execution)
|
|
584
|
+
transport_store.write(remote_archive, archive_bytes)
|
|
585
|
+
install_body = (
|
|
586
|
+
"from weaver.workspaces import FabricWorkspace\n"
|
|
587
|
+
"from weaver.build_bundle import InstallationEnvironment, "
|
|
588
|
+
"install_bundle_archive\n"
|
|
589
|
+
"from weaver.resolution import resolver_for, store_for\n"
|
|
590
|
+
f"workspace = {workspace_literal}\n"
|
|
591
|
+
"store = store_for(workspace)\n"
|
|
592
|
+
"resolver = resolver_for(workspace)\n"
|
|
593
|
+
"environment = InstallationEnvironment("
|
|
594
|
+
"store=store, resolver=resolver, spark=spark, workspace=workspace)\n"
|
|
595
|
+
f"archive = resolver.cli_bundle({execution_id!r})\n"
|
|
596
|
+
"report = install_bundle_archive(archive, archive_store=store, "
|
|
597
|
+
"environment=environment)\n"
|
|
598
|
+
"emit(report.to_mapping())\n"
|
|
599
|
+
)
|
|
600
|
+
report = session.run(install_body).payload
|
|
601
|
+
if retained_archive is not None:
|
|
602
|
+
transport_store.make_directory(resolver.build_bundles_root)
|
|
603
|
+
transport_store.write(retained_archive, archive_bytes)
|
|
604
|
+
finally:
|
|
605
|
+
try:
|
|
606
|
+
transport_store.delete(execution, recursive=True)
|
|
607
|
+
except WeaverError:
|
|
608
|
+
pass
|
|
609
|
+
assert bundle is not None and report is not None
|
|
610
|
+
return BuildResult(
|
|
611
|
+
source=source,
|
|
612
|
+
items=tuple(str(binding.item) for binding in bindings.entries),
|
|
613
|
+
bundle_id=bundle.bundle_id,
|
|
614
|
+
archive=retained_archive.value if retained_archive else None,
|
|
615
|
+
status=report["status"],
|
|
616
|
+
errors=tuple(
|
|
617
|
+
BuildFailure(
|
|
618
|
+
action["action_id"],
|
|
619
|
+
action.get("error_type"),
|
|
620
|
+
action.get("error_message"),
|
|
621
|
+
)
|
|
622
|
+
for sequence in report.get("sequences", ())
|
|
623
|
+
for action in sequence.get("actions", ())
|
|
624
|
+
if action.get("status") == "failed"
|
|
625
|
+
),
|
|
626
|
+
)
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def _binding_text(binding) -> str:
|
|
630
|
+
target = binding.target
|
|
631
|
+
if hasattr(target, "lakehouse"):
|
|
632
|
+
physical = f"Lakehouse/{target.lakehouse.name}"
|
|
633
|
+
else:
|
|
634
|
+
physical = f"Warehouse/{target.warehouse.name}"
|
|
635
|
+
return f"{physical}={binding.item}"
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _operation_store(workspace: Workspace) -> Store:
|
|
639
|
+
if isinstance(workspace, LocalWorkspace):
|
|
640
|
+
return LocalStore()
|
|
641
|
+
if _inside_fabric_session(workspace):
|
|
642
|
+
from .fabric.store import FabricStore
|
|
643
|
+
|
|
644
|
+
return FabricStore()
|
|
645
|
+
from .fabric import OneLakeDfsClient
|
|
646
|
+
|
|
647
|
+
return OneLakeDfsClient()
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _wipe_one(target: WipeTarget, workspace, *, store, dry_run):
|
|
651
|
+
from .physical_wipe import wipe_lakehouse, wipe_sql_target
|
|
652
|
+
|
|
653
|
+
if target.item_type == "Lakehouse":
|
|
654
|
+
low = wipe_lakehouse(target.item, workspace, store=store, dry_run=dry_run)
|
|
655
|
+
return tuple(
|
|
656
|
+
WipeReport(
|
|
657
|
+
target=str(target),
|
|
658
|
+
location=report.location,
|
|
659
|
+
removed=report.removed,
|
|
660
|
+
dry_run=dry_run,
|
|
661
|
+
)
|
|
662
|
+
for report in low
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
report = WipeReport(
|
|
666
|
+
target=str(target),
|
|
667
|
+
location=Location(f"warehouse://{target.item.name}"),
|
|
668
|
+
removed=("all user-created SQL objects",),
|
|
669
|
+
dry_run=dry_run,
|
|
670
|
+
)
|
|
671
|
+
if dry_run:
|
|
672
|
+
return (report,)
|
|
673
|
+
warehouse = WarehouseTarget(target.item)
|
|
674
|
+
if _inside_fabric_session(workspace):
|
|
675
|
+
wipe_sql_target(warehouse, workspace)
|
|
676
|
+
else:
|
|
677
|
+
from .fabric import desktop_sql_executor
|
|
678
|
+
|
|
679
|
+
with desktop_sql_executor(warehouse, workspace) as sql:
|
|
680
|
+
wipe_sql_target(warehouse, workspace, sql=sql)
|
|
681
|
+
return (report,)
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def _drop_local_catalogue(workspace, targets: Sequence[WipeTarget]) -> None:
|
|
685
|
+
if not isinstance(workspace, LocalWorkspace):
|
|
686
|
+
return
|
|
687
|
+
lakehouses = {target.item for target in targets}
|
|
688
|
+
if not lakehouses:
|
|
689
|
+
return
|
|
690
|
+
from .resolution import resolver_for
|
|
691
|
+
from .spark import drop_local_destination_catalogue, local_delta_session
|
|
692
|
+
|
|
693
|
+
resolver = resolver_for(workspace)
|
|
694
|
+
with local_delta_session(workspace) as session:
|
|
695
|
+
for lakehouse in lakehouses:
|
|
696
|
+
drop_local_destination_catalogue(
|
|
697
|
+
session, resolver.spark_destination(lakehouse)
|
|
698
|
+
)
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _unbind_physical_targets(workspace: Workspace, targets: Sequence[WipeTarget]):
|
|
702
|
+
lakehouses = sorted(
|
|
703
|
+
{target.physical_name for target in targets if target.item_type == "Lakehouse"}
|
|
704
|
+
)
|
|
705
|
+
warehouses = sorted(
|
|
706
|
+
{target.physical_name for target in targets if target.item_type == "Warehouse"}
|
|
707
|
+
)
|
|
708
|
+
if isinstance(workspace, LocalWorkspace) or _inside_fabric_session(workspace):
|
|
709
|
+
from .resolution import resolver_for
|
|
710
|
+
from .spark import SparkCatalogue
|
|
711
|
+
from .unbind import unbind_targets
|
|
712
|
+
|
|
713
|
+
if isinstance(workspace, LocalWorkspace):
|
|
714
|
+
from .spark import local_delta_session
|
|
715
|
+
|
|
716
|
+
with local_delta_session(workspace) as session:
|
|
717
|
+
catalogue = SparkCatalogue(
|
|
718
|
+
session,
|
|
719
|
+
resolver_for(workspace).spark_destination(
|
|
720
|
+
ItemRef(workspace.weaver_lakehouse)
|
|
721
|
+
),
|
|
722
|
+
)
|
|
723
|
+
return unbind_targets(
|
|
724
|
+
catalogue, lakehouses=lakehouses, warehouses=warehouses
|
|
725
|
+
).to_mapping()
|
|
726
|
+
catalogue = SparkCatalogue(
|
|
727
|
+
_active_spark(),
|
|
728
|
+
resolver_for(workspace).spark_destination(
|
|
729
|
+
ItemRef(workspace.weaver_lakehouse)
|
|
730
|
+
),
|
|
731
|
+
)
|
|
732
|
+
return unbind_targets(
|
|
733
|
+
catalogue, lakehouses=lakehouses, warehouses=warehouses
|
|
734
|
+
).to_mapping()
|
|
735
|
+
|
|
736
|
+
if not workspace.environment:
|
|
737
|
+
raise CommandError(
|
|
738
|
+
"Fabric catalogue unbind requires an Environment in workspace configuration"
|
|
739
|
+
)
|
|
740
|
+
from .fabric import LivySession
|
|
741
|
+
|
|
742
|
+
body = (
|
|
743
|
+
"from weaver.workspaces import FabricWorkspace\n"
|
|
744
|
+
"from weaver.targets import ItemRef\n"
|
|
745
|
+
"from weaver.unbind import unbind_targets\n"
|
|
746
|
+
"from weaver.resolution import resolver_for\n"
|
|
747
|
+
"from weaver.spark import SparkCatalogue\n"
|
|
748
|
+
f"workspace = FabricWorkspace(workspace={workspace.workspace!r}, "
|
|
749
|
+
f"environment={workspace.environment!r}, "
|
|
750
|
+
f"weaver_lakehouse={workspace.weaver_lakehouse!r})\n"
|
|
751
|
+
"catalogue = SparkCatalogue(spark, resolver_for(workspace).spark_destination("
|
|
752
|
+
"ItemRef(workspace.weaver_lakehouse)))\n"
|
|
753
|
+
f"emit(unbind_targets(catalogue, lakehouses={tuple(lakehouses)!r}, "
|
|
754
|
+
f"warehouses={tuple(warehouses)!r}).to_mapping())\n"
|
|
755
|
+
)
|
|
756
|
+
with LivySession.for_workspace(workspace) as session:
|
|
757
|
+
return session.run(body).payload
|