weaverstack 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. weaver/__init__.py +59 -0
  2. weaver/build_bundle/__init__.py +109 -0
  3. weaver/build_bundle/aliases.py +325 -0
  4. weaver/build_bundle/bundle.py +359 -0
  5. weaver/build_bundle/catalogue_actions.py +275 -0
  6. weaver/build_bundle/changes.py +186 -0
  7. weaver/build_bundle/endpoints.py +83 -0
  8. weaver/build_bundle/executors/__init__.py +69 -0
  9. weaver/build_bundle/executors/alias.py +202 -0
  10. weaver/build_bundle/executors/base.py +132 -0
  11. weaver/build_bundle/executors/folder.py +71 -0
  12. weaver/build_bundle/executors/load_file.py +205 -0
  13. weaver/build_bundle/executors/spark_case.py +26 -0
  14. weaver/build_bundle/executors/spark_schema.py +60 -0
  15. weaver/build_bundle/executors/spark_sql.py +59 -0
  16. weaver/build_bundle/executors/spark_sql_batch.py +57 -0
  17. weaver/build_bundle/executors/spark_table.py +213 -0
  18. weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
  19. weaver/build_bundle/executors/tsql.py +81 -0
  20. weaver/build_bundle/incremental.py +288 -0
  21. weaver/build_bundle/installer.py +384 -0
  22. weaver/build_bundle/models.py +288 -0
  23. weaver/build_bundle/payloads.py +34 -0
  24. weaver/build_bundle/physical.py +625 -0
  25. weaver/build_bundle/planner.py +389 -0
  26. weaver/build_bundle/prune.py +620 -0
  27. weaver/build_bundle/report.py +108 -0
  28. weaver/build_bundle/stages.py +196 -0
  29. weaver/build_bundle/targets.py +272 -0
  30. weaver/build_bundle/workflow.py +585 -0
  31. weaver/catalogue/__init__.py +73 -0
  32. weaver/catalogue/builtin.py +238 -0
  33. weaver/catalogue/claims.py +121 -0
  34. weaver/catalogue/projection.py +437 -0
  35. weaver/catalogue/reader.py +152 -0
  36. weaver/catalogue/reconcile.py +231 -0
  37. weaver/catalogue/render.py +410 -0
  38. weaver/catalogue/state.py +660 -0
  39. weaver/catalogue/tables.py +648 -0
  40. weaver/config.py +178 -0
  41. weaver/declaration/__init__.py +171 -0
  42. weaver/declaration/columns.py +223 -0
  43. weaver/declaration/ddl.py +266 -0
  44. weaver/declaration/dependencies.py +544 -0
  45. weaver/declaration/graph.py +240 -0
  46. weaver/declaration/item_dependencies.py +292 -0
  47. weaver/declaration/load.py +191 -0
  48. weaver/declaration/metadata.py +1405 -0
  49. weaver/declaration/model.py +448 -0
  50. weaver/declaration/references.py +294 -0
  51. weaver/declaration/repository.py +959 -0
  52. weaver/declaration/schemas.py +135 -0
  53. weaver/declaration/source.py +674 -0
  54. weaver/declaration/spark_load.py +759 -0
  55. weaver/declaration/sql_shaping.py +591 -0
  56. weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
  57. weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
  58. weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
  59. weaver/declaration/templates/load/column_metadata.sql +40 -0
  60. weaver/declaration/templates/load/full_replace_body.sql +21 -0
  61. weaver/declaration/templates/load/install_load_procedure.sql +27 -0
  62. weaver/declaration/templates/load/load_procedure.sql +48 -0
  63. weaver/declaration/templates/load/primary_key_body.sql +113 -0
  64. weaver/declaration/tsql_ddl.py +468 -0
  65. weaver/declaration/tsql_load.py +417 -0
  66. weaver/declaration/warehouse_type_mapping.yml +93 -0
  67. weaver/diagnostics.py +247 -0
  68. weaver/errors.py +61 -0
  69. weaver/etl.py +469 -0
  70. weaver/fabric/__init__.py +107 -0
  71. weaver/fabric/auth.py +137 -0
  72. weaver/fabric/capacity.py +143 -0
  73. weaver/fabric/client.py +147 -0
  74. weaver/fabric/environment.py +460 -0
  75. weaver/fabric/livy.py +478 -0
  76. weaver/fabric/notebooks.py +201 -0
  77. weaver/fabric/onelake.py +263 -0
  78. weaver/fabric/resolution.py +344 -0
  79. weaver/fabric/resources.py +245 -0
  80. weaver/fabric/session.py +148 -0
  81. weaver/fabric/shortcuts.py +120 -0
  82. weaver/fabric/sql.py +118 -0
  83. weaver/fabric/store.py +198 -0
  84. weaver/initialise.py +209 -0
  85. weaver/lakehouse.py +386 -0
  86. weaver/load.py +474 -0
  87. weaver/load_execution.py +483 -0
  88. weaver/load_plan.py +912 -0
  89. weaver/load_report.py +330 -0
  90. weaver/load_resolution.py +386 -0
  91. weaver/locations.py +164 -0
  92. weaver/objects.py +392 -0
  93. weaver/operations.py +757 -0
  94. weaver/physical_wipe.py +369 -0
  95. weaver/push.py +76 -0
  96. weaver/resolution.py +292 -0
  97. weaver/runtime/__init__.py +30 -0
  98. weaver/runtime/folder_load.py +402 -0
  99. weaver/runtime/load_contract.py +245 -0
  100. weaver/runtime/load_result.py +104 -0
  101. weaver/runtime/spark_load.py +152 -0
  102. weaver/runtime/table_load.py +497 -0
  103. weaver/spark/__init__.py +49 -0
  104. weaver/spark/catalogue.py +245 -0
  105. weaver/spark/destination.py +195 -0
  106. weaver/spark/session.py +84 -0
  107. weaver/spark/tokens.py +138 -0
  108. weaver/sql/__init__.py +40 -0
  109. weaver/sql/authentication.py +38 -0
  110. weaver/sql/connection.py +90 -0
  111. weaver/sql/errors.py +25 -0
  112. weaver/sql/execution.py +123 -0
  113. weaver/sql/pool.py +174 -0
  114. weaver/sql/wipe.py +156 -0
  115. weaver/store.py +209 -0
  116. weaver/targets.py +257 -0
  117. weaver/task_logging.py +215 -0
  118. weaver/unbind.py +74 -0
  119. weaver/workspaces.py +175 -0
  120. weaver_cli/__init__.py +12 -0
  121. weaver_cli/__main__.py +7 -0
  122. weaver_cli/main.py +626 -0
  123. weaverstack-0.1.1.dist-info/METADATA +113 -0
  124. weaverstack-0.1.1.dist-info/RECORD +127 -0
  125. weaverstack-0.1.1.dist-info/WHEEL +4 -0
  126. weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
  127. weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
weaver/operations.py ADDED
@@ -0,0 +1,757 @@
1
+ """Source-neutral public build and target-oriented wipe operations.
2
+
3
+ This module is the adaptation boundary between the small notebook-facing API
4
+ and Weaver's typed planning and execution machinery. It deliberately keeps
5
+ all optional platform imports inside the path that needs them so importing
6
+ ``weaver`` remains safe without Spark, Fabric credentials, or the desktop CLI.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass, replace
12
+ from pathlib import Path
13
+ import tempfile
14
+ import uuid
15
+ from typing import Iterable, Mapping, Sequence
16
+
17
+ from .errors import BuildError, CommandError, WeaverError
18
+ from .locations import Location
19
+ from .store import LocalStore, Store
20
+ from .targets import (
21
+ ItemRef,
22
+ WarehouseTarget,
23
+ parse_physical_target,
24
+ physical_item,
25
+ physical_kind,
26
+ )
27
+ from .workspaces import FabricWorkspace, LocalWorkspace, Workspace
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class BuildFailure:
32
+ action_id: str
33
+ error_type: str | None
34
+ message: str | None
35
+
36
+ def to_mapping(self) -> dict:
37
+ return {
38
+ "id": self.action_id,
39
+ "type": self.error_type,
40
+ "message": self.message,
41
+ }
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class BuildResult:
46
+ source: str
47
+ items: tuple[str, ...]
48
+ bundle_id: str
49
+ archive: str | None
50
+ status: str
51
+ errors: tuple[BuildFailure, ...] = ()
52
+
53
+ @property
54
+ def succeeded(self) -> bool:
55
+ return self.status == "succeeded"
56
+
57
+ def to_mapping(self) -> dict:
58
+ return {
59
+ "source": self.source,
60
+ "items": list(self.items),
61
+ "bundle_id": self.bundle_id,
62
+ "archive": self.archive,
63
+ "status": self.status,
64
+ "errors": [error.to_mapping() for error in self.errors],
65
+ }
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class WipeTarget:
70
+ item_type: str
71
+ item: ItemRef
72
+
73
+ @classmethod
74
+ def parse(cls, text: str) -> "WipeTarget":
75
+ target = parse_physical_target(
76
+ text, what="wipe target", error=CommandError
77
+ )
78
+ return cls(item_type=physical_kind(target), item=physical_item(target))
79
+
80
+ @property
81
+ def physical_name(self) -> str:
82
+ return self.item.name
83
+
84
+ def __str__(self) -> str:
85
+ return f"{self.item_type}/{self.item}"
86
+
87
+
88
+ def _unbind_target_names(targets: Iterable[str]) -> tuple[tuple[str, ...], tuple[str, ...]]:
89
+ """Parse unbind selection through the same typed grammar used by wipe."""
90
+
91
+ parsed = tuple(WipeTarget.parse(target) for target in targets)
92
+ return (
93
+ tuple(target.physical_name for target in parsed if target.item_type == "Lakehouse"),
94
+ tuple(target.physical_name for target in parsed if target.item_type == "Warehouse"),
95
+ )
96
+
97
+
98
+ @dataclass(frozen=True)
99
+ class WipeReport:
100
+ target: str
101
+ location: Location
102
+ removed: tuple[str, ...]
103
+ dry_run: bool = False
104
+
105
+ @property
106
+ def count(self) -> int:
107
+ return len(self.removed)
108
+
109
+ def to_mapping(self) -> dict:
110
+ return {
111
+ "target": self.target,
112
+ "location": self.location.value,
113
+ "removed": list(self.removed),
114
+ "dry_run": self.dry_run,
115
+ }
116
+
117
+
118
+ @dataclass(frozen=True)
119
+ class WipeResult:
120
+ workspace: str
121
+ reports: tuple[WipeReport, ...]
122
+ unbound: Mapping | None = None
123
+ dry_run: bool = False
124
+
125
+ @property
126
+ def count(self) -> int:
127
+ return sum(report.count for report in self.reports)
128
+
129
+ def to_mapping(self) -> dict:
130
+ return {
131
+ "workspace": self.workspace,
132
+ "reports": [report.to_mapping() for report in self.reports],
133
+ "unbound": dict(self.unbound) if self.unbound is not None else None,
134
+ "dry_run": self.dry_run,
135
+ }
136
+
137
+
138
+ def build(
139
+ source=None,
140
+ *,
141
+ bind: str | Sequence[str] | None = None,
142
+ workspace: str | Path | Workspace | None = None,
143
+ workspace_config: str | Path | None = None,
144
+ bundle: str | None = None,
145
+ ) -> BuildResult:
146
+ """Build an authored repository using simple notebook-facing values.
147
+
148
+ ``workspace=None`` means the current Fabric session. A typed ``Workspace``
149
+ remains accepted as an advanced/testing seam and is what the CLI supplies
150
+ after applying its explicit ``--workspace-type`` flags.
151
+ """
152
+
153
+ resolved_workspace = _operation_workspace(
154
+ workspace=workspace, workspace_config=workspace_config
155
+ )
156
+ resolved_workspace = _with_inferred_control_lakehouse(resolved_workspace)
157
+ if not resolved_workspace.weaver_lakehouse:
158
+ raise CommandError(
159
+ "build needs a Weaver control Lakehouse in workspace configuration "
160
+ "or as the notebook's attached default Lakehouse"
161
+ )
162
+
163
+ selected = _item_bindings(bind, resolved_workspace)
164
+ from .build_bundle.targets import LakehouseBinding, effective_item_bindings
165
+
166
+ bindings = effective_item_bindings(
167
+ selected, weaver_lakehouse=resolved_workspace.weaver_lakehouse
168
+ )
169
+ control = LakehouseBinding(ItemRef(resolved_workspace.weaver_lakehouse))
170
+ source_location, source_store = _repository_source(source, resolved_workspace)
171
+
172
+ # This complete parse and pure request validation is deliberately above all
173
+ # control-plane creation, Spark start, REST item resolution, and Livy work.
174
+ from .build_bundle.workflow import prepare_repository, validate_build_request
175
+
176
+ with prepare_repository(
177
+ source_location, source_store=source_store
178
+ ) as prepared:
179
+ validate_build_request(
180
+ prepared.repository, bindings, control_lakehouse=control
181
+ )
182
+ if isinstance(resolved_workspace, LocalWorkspace):
183
+ return _build_local(
184
+ resolved_workspace,
185
+ repository=prepared.repository,
186
+ source_store=prepared.store,
187
+ bindings=bindings,
188
+ control_lakehouse=control,
189
+ bundle_name=bundle,
190
+ source=source_location.value,
191
+ )
192
+ if _inside_fabric_session(resolved_workspace):
193
+ return _build_native_fabric(
194
+ resolved_workspace,
195
+ repository=prepared.repository,
196
+ source_store=prepared.store,
197
+ bindings=bindings,
198
+ control_lakehouse=control,
199
+ bundle_name=bundle,
200
+ source=source_location.value,
201
+ )
202
+ return _build_desktop_fabric(
203
+ resolved_workspace,
204
+ repository=prepared.repository,
205
+ source_store=prepared.store,
206
+ bindings=bindings,
207
+ control_lakehouse=control,
208
+ bundle_name=bundle,
209
+ source=source_location.value,
210
+ )
211
+
212
+
213
+ def wipe(
214
+ targets: str | Iterable[str],
215
+ *,
216
+ workspace: str | Path | Workspace | None = None,
217
+ workspace_config: str | Path | None = None,
218
+ unbind_from: str | None = None,
219
+ dry_run: bool = False,
220
+ ) -> WipeResult:
221
+ """Empty one or more whole Lakehouse or Warehouse items."""
222
+
223
+ values = (targets,) if isinstance(targets, str) else tuple(targets)
224
+ parsed = tuple(WipeTarget.parse(value) for value in values)
225
+ if not parsed:
226
+ raise CommandError("wipe needs at least one target")
227
+ resolved_workspace = _operation_workspace(
228
+ workspace=workspace, workspace_config=workspace_config
229
+ )
230
+ if isinstance(resolved_workspace, LocalWorkspace) and any(
231
+ target.item_type == "Warehouse" for target in parsed
232
+ ):
233
+ raise CommandError(
234
+ "Warehouse targets require a Fabric Workspace; the local emulator has no SQL"
235
+ )
236
+
237
+ storage_targets = tuple(t for t in parsed if t.item_type == "Lakehouse")
238
+ store = _operation_store(resolved_workspace) if storage_targets else None
239
+ reports: list[WipeReport] = []
240
+ if not dry_run:
241
+ _drop_local_catalogue(resolved_workspace, storage_targets)
242
+ for target in parsed:
243
+ reports.extend(
244
+ _wipe_one(target, resolved_workspace, store=store, dry_run=dry_run)
245
+ )
246
+
247
+ unbound = None
248
+ control = unbind_from or resolved_workspace.weaver_lakehouse
249
+ whole_lakehouses = {
250
+ target.physical_name
251
+ for target in parsed
252
+ if target.item_type == "Lakehouse"
253
+ }
254
+ if not dry_run and control and control not in whole_lakehouses:
255
+ catalogue_workspace = replace(
256
+ resolved_workspace, weaver_lakehouse=ItemRef.parse(control).name
257
+ )
258
+ unbound = _unbind_physical_targets(catalogue_workspace, parsed)
259
+
260
+ return WipeResult(
261
+ workspace=str(resolved_workspace.workspace),
262
+ reports=tuple(reports),
263
+ unbound=unbound,
264
+ dry_run=dry_run,
265
+ )
266
+
267
+
268
+ def _operation_workspace(*, workspace, workspace_config) -> Workspace:
269
+ if isinstance(workspace, Workspace):
270
+ if workspace_config is not None:
271
+ raise CommandError(
272
+ "workspace_config cannot be combined with an already resolved Workspace"
273
+ )
274
+ return workspace
275
+ if workspace is None and workspace_config is None:
276
+ return _current_fabric_workspace()
277
+ from .config import resolve_workspace
278
+
279
+ return resolve_workspace(workspace=workspace, workspace_config=workspace_config)
280
+
281
+
282
+ def _current_fabric_workspace() -> FabricWorkspace:
283
+ try:
284
+ from notebookutils import runtime
285
+ except ImportError as exc:
286
+ raise CommandError(
287
+ "give workspace or workspace_config outside a Fabric notebook"
288
+ ) from exc
289
+ context = runtime.context
290
+ if callable(context):
291
+ context = context()
292
+ if not isinstance(context, Mapping):
293
+ raise CommandError("Fabric runtime context is not a mapping")
294
+ name = context.get("currentWorkspaceName")
295
+ if not name:
296
+ raise CommandError("Fabric runtime context carries no current workspace")
297
+ return FabricWorkspace(workspace=str(name))
298
+
299
+
300
+ def _with_inferred_control_lakehouse(workspace: Workspace) -> Workspace:
301
+ if workspace.weaver_lakehouse or not isinstance(workspace, FabricWorkspace):
302
+ return workspace
303
+ if not _inside_fabric_session(workspace):
304
+ return workspace
305
+ from .lakehouse import default_lakehouse
306
+
307
+ spark = _active_spark()
308
+ return replace(workspace, weaver_lakehouse=default_lakehouse(spark).name)
309
+
310
+
311
+ def _active_spark():
312
+ try:
313
+ from importlib import import_module
314
+
315
+ SparkSession = import_module("pyspark.sql").SparkSession
316
+ except ImportError as exc:
317
+ raise CommandError("this operation needs an active Spark session") from exc
318
+ spark = SparkSession.getActiveSession()
319
+ if spark is None:
320
+ raise CommandError("this operation needs an active Spark session")
321
+ return spark
322
+
323
+
324
+ def _inside_fabric_session(workspace: FabricWorkspace) -> bool:
325
+ try:
326
+ from notebookutils import runtime
327
+ except ImportError:
328
+ return False
329
+ context = runtime.context
330
+ if callable(context):
331
+ context = context()
332
+ if not isinstance(context, Mapping):
333
+ return False
334
+ return context.get("currentWorkspaceName") == workspace.workspace
335
+
336
+
337
+ def _repository_source(source, workspace: Workspace) -> tuple[Location, Store]:
338
+ if source is None:
339
+ if not isinstance(workspace, FabricWorkspace) or not _inside_fabric_session(workspace):
340
+ source = "."
341
+ else:
342
+ # Fabric exposes built-in Notebook Resources as the notebook's
343
+ # process-local working tree. No OneLake adapter is involved.
344
+ source = Path.cwd()
345
+ location = source if isinstance(source, Location) else Location(str(source))
346
+ if location.value.startswith("abfss://"):
347
+ if not isinstance(workspace, FabricWorkspace) or not _inside_fabric_session(workspace):
348
+ raise CommandError(
349
+ "an abfss repository source can be read only inside a Fabric session"
350
+ )
351
+ from .fabric.store import FabricStore
352
+
353
+ return location, FabricStore()
354
+ return location, LocalStore()
355
+
356
+
357
+ def _item_bindings(bind, workspace: Workspace):
358
+ from .build_bundle.targets import ItemBindings, parse_item_binding
359
+
360
+ if bind is None:
361
+ values = [f"Lakehouse/{name}" for name in workspace.lakehouses]
362
+ values += [f"Warehouse/{name}" for name in workspace.warehouses]
363
+ elif isinstance(bind, str):
364
+ values = [bind]
365
+ else:
366
+ values = list(bind)
367
+ if not values:
368
+ raise BuildError(
369
+ "build needs bind values or configured Lakehouse/Warehouse targets"
370
+ )
371
+ return ItemBindings(
372
+ tuple(parse_item_binding(value, workspace=workspace) for value in values)
373
+ )
374
+
375
+
376
+ def _ensure_control_plane(workspace: Workspace, *, store: Store, spark) -> None:
377
+ from .initialise import initialise_weaver_lakehouse, prepare_weaver_lakehouse
378
+
379
+ resolver = None
380
+ client = None
381
+ if isinstance(workspace, FabricWorkspace) and _inside_fabric_session(workspace):
382
+ from .resolution import resolver_for
383
+
384
+ resolver = resolver_for(workspace)
385
+ factory = getattr(resolver, "_rest_client", None)
386
+ client = factory() if callable(factory) else None
387
+ prepare_weaver_lakehouse(workspace, exists_ok=True, store=store, client=client)
388
+ initialise_weaver_lakehouse(
389
+ weaver_lakehouse=ItemRef(workspace.weaver_lakehouse),
390
+ workspace=workspace,
391
+ store=store,
392
+ spark=spark,
393
+ )
394
+
395
+
396
+ def _archive_location(resolver, bundle_name: str | None):
397
+ if bundle_name is None:
398
+ return None
399
+ from .build_bundle.workflow import timestamped_archive_name
400
+
401
+ name = bundle_name or timestamped_archive_name()
402
+ if not name.endswith(".weaver.zip"):
403
+ name += ".weaver.zip"
404
+ return resolver.build_bundle(name)
405
+
406
+
407
+ def _result_from_item_build(source, bindings, result) -> BuildResult:
408
+ report = result.report
409
+ return BuildResult(
410
+ source=source,
411
+ items=tuple(str(binding.item) for binding in bindings.entries),
412
+ bundle_id=result.bundle_id,
413
+ archive=result.archive.value if result.archive else None,
414
+ status=report.status,
415
+ errors=tuple(
416
+ BuildFailure(
417
+ action.action_id, action.error_type, action.error_message
418
+ )
419
+ for action in report.action_results()
420
+ if action.status == "failed"
421
+ ),
422
+ )
423
+
424
+
425
+ def _build_in_process(
426
+ workspace,
427
+ *,
428
+ spark,
429
+ store,
430
+ repository,
431
+ source_store,
432
+ bindings,
433
+ control_lakehouse,
434
+ bundle_name,
435
+ source,
436
+ ) -> BuildResult:
437
+ from .build_bundle import (
438
+ InstallationEnvironment,
439
+ build_item_repository,
440
+ catalogue_items_for_build,
441
+ read_build_state,
442
+ )
443
+ from .catalogue.state import reconcile_catalogue_state
444
+ from .resolution import resolver_for
445
+
446
+ _ensure_control_plane(workspace, store=store, spark=spark)
447
+ resolver = resolver_for(workspace)
448
+ environment = InstallationEnvironment(
449
+ store=store, resolver=resolver, spark=spark, workspace=workspace
450
+ )
451
+ state = read_build_state(
452
+ bindings,
453
+ required_catalogue_items=catalogue_items_for_build(repository, bindings),
454
+ environment=environment,
455
+ )
456
+ result = build_item_repository(
457
+ repository,
458
+ bindings=bindings,
459
+ target_inventories=state.target_inventories,
460
+ reconciliation=reconcile_catalogue_state(
461
+ state.catalogue, inventories=state.target_inventories
462
+ ),
463
+ environment=environment,
464
+ source_store=source_store,
465
+ control_lakehouse=control_lakehouse,
466
+ archive=_archive_location(resolver, bundle_name),
467
+ )
468
+ return _result_from_item_build(source, bindings, result)
469
+
470
+
471
+ def _build_local(workspace, **kwargs) -> BuildResult:
472
+ warehouse_items = [
473
+ str(binding.item)
474
+ for binding in kwargs["bindings"].entries
475
+ if binding.item.item_type == "Warehouse"
476
+ ]
477
+ if warehouse_items:
478
+ raise CommandError(
479
+ "local Workspace builds cannot target Warehouses: "
480
+ + ", ".join(warehouse_items)
481
+ )
482
+ from .spark import local_delta_session
483
+
484
+ with local_delta_session(workspace) as session:
485
+ return _build_in_process(
486
+ workspace, spark=session, store=LocalStore(), **kwargs
487
+ )
488
+
489
+
490
+ def _build_native_fabric(workspace, **kwargs) -> BuildResult:
491
+ from .resolution import store_for
492
+
493
+ return _build_in_process(
494
+ workspace, spark=_active_spark(), store=store_for(workspace), **kwargs
495
+ )
496
+
497
+
498
+ def _build_desktop_fabric(
499
+ workspace,
500
+ *,
501
+ repository,
502
+ source_store,
503
+ bindings,
504
+ control_lakehouse,
505
+ bundle_name,
506
+ source,
507
+ ) -> BuildResult:
508
+ if not workspace.environment:
509
+ raise CommandError(
510
+ "Fabric build requires an Environment in workspace configuration"
511
+ )
512
+ from .build_bundle import (
513
+ BuildState,
514
+ catalogue_items_for_build,
515
+ generate_item_build_bundle,
516
+ persist_bundle_archive,
517
+ )
518
+ from .catalogue.state import reconcile_catalogue_state
519
+ from .fabric import LivySession, OneLakeDfsClient
520
+ from .initialise import prepare_weaver_lakehouse
521
+ from .resolution import resolver_for
522
+
523
+ prepare_weaver_lakehouse(workspace, exists_ok=True)
524
+ resolver = resolver_for(workspace)
525
+ transport_store = OneLakeDfsClient()
526
+ binding_texts = [_binding_text(binding) for binding in bindings.entries]
527
+ required_items = [
528
+ str(item) for item in catalogue_items_for_build(repository, bindings)
529
+ ]
530
+ workspace_literal = (
531
+ f"FabricWorkspace(workspace={workspace.workspace!r}, "
532
+ f"weaver_lakehouse={workspace.weaver_lakehouse!r}, "
533
+ f"environment={workspace.environment!r})"
534
+ )
535
+ state_body = (
536
+ "from weaver.workspaces import FabricWorkspace\n"
537
+ "from weaver.declaration.model import WeaverItemId\n"
538
+ "from weaver.build_bundle import (InstallationEnvironment, ItemBindings, "
539
+ "parse_item_binding, read_build_state)\n"
540
+ "from weaver.initialise import initialise_weaver_lakehouse\n"
541
+ "from weaver.targets import ItemRef\n"
542
+ "from weaver.resolution import resolver_for, store_for\n"
543
+ f"workspace = {workspace_literal}\n"
544
+ "store = store_for(workspace)\n"
545
+ "resolver = resolver_for(workspace)\n"
546
+ "initialise_weaver_lakehouse(weaver_lakehouse=ItemRef("
547
+ "workspace.weaver_lakehouse), workspace=workspace, store=store, spark=spark)\n"
548
+ f"bindings = ItemBindings(tuple(parse_item_binding(text) for text in {binding_texts!r}))\n"
549
+ "environment = InstallationEnvironment("
550
+ "store=store, resolver=resolver, spark=spark, workspace=workspace)\n"
551
+ f"items = tuple(WeaverItemId.parse(value) for value in {required_items!r})\n"
552
+ "emit(read_build_state(bindings, required_catalogue_items=items, "
553
+ "environment=environment).to_mapping())\n"
554
+ )
555
+ execution_id = uuid.uuid4().hex
556
+ execution = resolver.cli_execution(execution_id)
557
+ remote_archive = resolver.cli_bundle(execution_id)
558
+ retained_archive = _archive_location(resolver, bundle_name)
559
+ bundle = None
560
+ report = None
561
+ with LivySession.for_workspace(workspace) as session:
562
+ state = BuildState.from_mapping(session.run(state_body).payload)
563
+ reconciliation = reconcile_catalogue_state(
564
+ state.catalogue, inventories=state.target_inventories
565
+ )
566
+ try:
567
+ with tempfile.TemporaryDirectory(prefix="weaver-cli-build-") as temporary:
568
+ root = Path(temporary)
569
+ bundle = generate_item_build_bundle(
570
+ repository,
571
+ bindings=bindings,
572
+ output=Location((root / "bundle").as_posix()),
573
+ store=source_store,
574
+ target_inventories=state.target_inventories,
575
+ catalogue=reconciliation.catalogue,
576
+ stale_claims=reconciliation.stale_claims,
577
+ control_lakehouse=control_lakehouse,
578
+ )
579
+ local_archive = Location((root / "install.weaver.zip").as_posix())
580
+ persist_bundle_archive(bundle, local_archive, store=LocalStore())
581
+ archive_bytes = LocalStore().read(local_archive)
582
+ transport_store.make_directory(resolver.cli_root)
583
+ transport_store.make_directory(execution)
584
+ transport_store.write(remote_archive, archive_bytes)
585
+ install_body = (
586
+ "from weaver.workspaces import FabricWorkspace\n"
587
+ "from weaver.build_bundle import InstallationEnvironment, "
588
+ "install_bundle_archive\n"
589
+ "from weaver.resolution import resolver_for, store_for\n"
590
+ f"workspace = {workspace_literal}\n"
591
+ "store = store_for(workspace)\n"
592
+ "resolver = resolver_for(workspace)\n"
593
+ "environment = InstallationEnvironment("
594
+ "store=store, resolver=resolver, spark=spark, workspace=workspace)\n"
595
+ f"archive = resolver.cli_bundle({execution_id!r})\n"
596
+ "report = install_bundle_archive(archive, archive_store=store, "
597
+ "environment=environment)\n"
598
+ "emit(report.to_mapping())\n"
599
+ )
600
+ report = session.run(install_body).payload
601
+ if retained_archive is not None:
602
+ transport_store.make_directory(resolver.build_bundles_root)
603
+ transport_store.write(retained_archive, archive_bytes)
604
+ finally:
605
+ try:
606
+ transport_store.delete(execution, recursive=True)
607
+ except WeaverError:
608
+ pass
609
+ assert bundle is not None and report is not None
610
+ return BuildResult(
611
+ source=source,
612
+ items=tuple(str(binding.item) for binding in bindings.entries),
613
+ bundle_id=bundle.bundle_id,
614
+ archive=retained_archive.value if retained_archive else None,
615
+ status=report["status"],
616
+ errors=tuple(
617
+ BuildFailure(
618
+ action["action_id"],
619
+ action.get("error_type"),
620
+ action.get("error_message"),
621
+ )
622
+ for sequence in report.get("sequences", ())
623
+ for action in sequence.get("actions", ())
624
+ if action.get("status") == "failed"
625
+ ),
626
+ )
627
+
628
+
629
+ def _binding_text(binding) -> str:
630
+ target = binding.target
631
+ if hasattr(target, "lakehouse"):
632
+ physical = f"Lakehouse/{target.lakehouse.name}"
633
+ else:
634
+ physical = f"Warehouse/{target.warehouse.name}"
635
+ return f"{physical}={binding.item}"
636
+
637
+
638
+ def _operation_store(workspace: Workspace) -> Store:
639
+ if isinstance(workspace, LocalWorkspace):
640
+ return LocalStore()
641
+ if _inside_fabric_session(workspace):
642
+ from .fabric.store import FabricStore
643
+
644
+ return FabricStore()
645
+ from .fabric import OneLakeDfsClient
646
+
647
+ return OneLakeDfsClient()
648
+
649
+
650
+ def _wipe_one(target: WipeTarget, workspace, *, store, dry_run):
651
+ from .physical_wipe import wipe_lakehouse, wipe_sql_target
652
+
653
+ if target.item_type == "Lakehouse":
654
+ low = wipe_lakehouse(target.item, workspace, store=store, dry_run=dry_run)
655
+ return tuple(
656
+ WipeReport(
657
+ target=str(target),
658
+ location=report.location,
659
+ removed=report.removed,
660
+ dry_run=dry_run,
661
+ )
662
+ for report in low
663
+ )
664
+
665
+ report = WipeReport(
666
+ target=str(target),
667
+ location=Location(f"warehouse://{target.item.name}"),
668
+ removed=("all user-created SQL objects",),
669
+ dry_run=dry_run,
670
+ )
671
+ if dry_run:
672
+ return (report,)
673
+ warehouse = WarehouseTarget(target.item)
674
+ if _inside_fabric_session(workspace):
675
+ wipe_sql_target(warehouse, workspace)
676
+ else:
677
+ from .fabric import desktop_sql_executor
678
+
679
+ with desktop_sql_executor(warehouse, workspace) as sql:
680
+ wipe_sql_target(warehouse, workspace, sql=sql)
681
+ return (report,)
682
+
683
+
684
+ def _drop_local_catalogue(workspace, targets: Sequence[WipeTarget]) -> None:
685
+ if not isinstance(workspace, LocalWorkspace):
686
+ return
687
+ lakehouses = {target.item for target in targets}
688
+ if not lakehouses:
689
+ return
690
+ from .resolution import resolver_for
691
+ from .spark import drop_local_destination_catalogue, local_delta_session
692
+
693
+ resolver = resolver_for(workspace)
694
+ with local_delta_session(workspace) as session:
695
+ for lakehouse in lakehouses:
696
+ drop_local_destination_catalogue(
697
+ session, resolver.spark_destination(lakehouse)
698
+ )
699
+
700
+
701
+ def _unbind_physical_targets(workspace: Workspace, targets: Sequence[WipeTarget]):
702
+ lakehouses = sorted(
703
+ {target.physical_name for target in targets if target.item_type == "Lakehouse"}
704
+ )
705
+ warehouses = sorted(
706
+ {target.physical_name for target in targets if target.item_type == "Warehouse"}
707
+ )
708
+ if isinstance(workspace, LocalWorkspace) or _inside_fabric_session(workspace):
709
+ from .resolution import resolver_for
710
+ from .spark import SparkCatalogue
711
+ from .unbind import unbind_targets
712
+
713
+ if isinstance(workspace, LocalWorkspace):
714
+ from .spark import local_delta_session
715
+
716
+ with local_delta_session(workspace) as session:
717
+ catalogue = SparkCatalogue(
718
+ session,
719
+ resolver_for(workspace).spark_destination(
720
+ ItemRef(workspace.weaver_lakehouse)
721
+ ),
722
+ )
723
+ return unbind_targets(
724
+ catalogue, lakehouses=lakehouses, warehouses=warehouses
725
+ ).to_mapping()
726
+ catalogue = SparkCatalogue(
727
+ _active_spark(),
728
+ resolver_for(workspace).spark_destination(
729
+ ItemRef(workspace.weaver_lakehouse)
730
+ ),
731
+ )
732
+ return unbind_targets(
733
+ catalogue, lakehouses=lakehouses, warehouses=warehouses
734
+ ).to_mapping()
735
+
736
+ if not workspace.environment:
737
+ raise CommandError(
738
+ "Fabric catalogue unbind requires an Environment in workspace configuration"
739
+ )
740
+ from .fabric import LivySession
741
+
742
+ body = (
743
+ "from weaver.workspaces import FabricWorkspace\n"
744
+ "from weaver.targets import ItemRef\n"
745
+ "from weaver.unbind import unbind_targets\n"
746
+ "from weaver.resolution import resolver_for\n"
747
+ "from weaver.spark import SparkCatalogue\n"
748
+ f"workspace = FabricWorkspace(workspace={workspace.workspace!r}, "
749
+ f"environment={workspace.environment!r}, "
750
+ f"weaver_lakehouse={workspace.weaver_lakehouse!r})\n"
751
+ "catalogue = SparkCatalogue(spark, resolver_for(workspace).spark_destination("
752
+ "ItemRef(workspace.weaver_lakehouse)))\n"
753
+ f"emit(unbind_targets(catalogue, lakehouses={tuple(lakehouses)!r}, "
754
+ f"warehouses={tuple(warehouses)!r}).to_mapping())\n"
755
+ )
756
+ with LivySession.for_workspace(workspace) as session:
757
+ return session.run(body).payload