weaverstack 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. weaver/__init__.py +59 -0
  2. weaver/build_bundle/__init__.py +109 -0
  3. weaver/build_bundle/aliases.py +325 -0
  4. weaver/build_bundle/bundle.py +359 -0
  5. weaver/build_bundle/catalogue_actions.py +275 -0
  6. weaver/build_bundle/changes.py +186 -0
  7. weaver/build_bundle/endpoints.py +83 -0
  8. weaver/build_bundle/executors/__init__.py +69 -0
  9. weaver/build_bundle/executors/alias.py +202 -0
  10. weaver/build_bundle/executors/base.py +132 -0
  11. weaver/build_bundle/executors/folder.py +71 -0
  12. weaver/build_bundle/executors/load_file.py +205 -0
  13. weaver/build_bundle/executors/spark_case.py +26 -0
  14. weaver/build_bundle/executors/spark_schema.py +60 -0
  15. weaver/build_bundle/executors/spark_sql.py +59 -0
  16. weaver/build_bundle/executors/spark_sql_batch.py +57 -0
  17. weaver/build_bundle/executors/spark_table.py +213 -0
  18. weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
  19. weaver/build_bundle/executors/tsql.py +81 -0
  20. weaver/build_bundle/incremental.py +288 -0
  21. weaver/build_bundle/installer.py +384 -0
  22. weaver/build_bundle/models.py +288 -0
  23. weaver/build_bundle/payloads.py +34 -0
  24. weaver/build_bundle/physical.py +625 -0
  25. weaver/build_bundle/planner.py +389 -0
  26. weaver/build_bundle/prune.py +620 -0
  27. weaver/build_bundle/report.py +108 -0
  28. weaver/build_bundle/stages.py +196 -0
  29. weaver/build_bundle/targets.py +272 -0
  30. weaver/build_bundle/workflow.py +585 -0
  31. weaver/catalogue/__init__.py +73 -0
  32. weaver/catalogue/builtin.py +238 -0
  33. weaver/catalogue/claims.py +121 -0
  34. weaver/catalogue/projection.py +437 -0
  35. weaver/catalogue/reader.py +152 -0
  36. weaver/catalogue/reconcile.py +231 -0
  37. weaver/catalogue/render.py +410 -0
  38. weaver/catalogue/state.py +660 -0
  39. weaver/catalogue/tables.py +648 -0
  40. weaver/config.py +178 -0
  41. weaver/declaration/__init__.py +171 -0
  42. weaver/declaration/columns.py +223 -0
  43. weaver/declaration/ddl.py +266 -0
  44. weaver/declaration/dependencies.py +544 -0
  45. weaver/declaration/graph.py +240 -0
  46. weaver/declaration/item_dependencies.py +292 -0
  47. weaver/declaration/load.py +191 -0
  48. weaver/declaration/metadata.py +1405 -0
  49. weaver/declaration/model.py +448 -0
  50. weaver/declaration/references.py +294 -0
  51. weaver/declaration/repository.py +959 -0
  52. weaver/declaration/schemas.py +135 -0
  53. weaver/declaration/source.py +674 -0
  54. weaver/declaration/spark_load.py +759 -0
  55. weaver/declaration/sql_shaping.py +591 -0
  56. weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
  57. weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
  58. weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
  59. weaver/declaration/templates/load/column_metadata.sql +40 -0
  60. weaver/declaration/templates/load/full_replace_body.sql +21 -0
  61. weaver/declaration/templates/load/install_load_procedure.sql +27 -0
  62. weaver/declaration/templates/load/load_procedure.sql +48 -0
  63. weaver/declaration/templates/load/primary_key_body.sql +113 -0
  64. weaver/declaration/tsql_ddl.py +468 -0
  65. weaver/declaration/tsql_load.py +417 -0
  66. weaver/declaration/warehouse_type_mapping.yml +93 -0
  67. weaver/diagnostics.py +247 -0
  68. weaver/errors.py +61 -0
  69. weaver/etl.py +469 -0
  70. weaver/fabric/__init__.py +107 -0
  71. weaver/fabric/auth.py +137 -0
  72. weaver/fabric/capacity.py +143 -0
  73. weaver/fabric/client.py +147 -0
  74. weaver/fabric/environment.py +460 -0
  75. weaver/fabric/livy.py +478 -0
  76. weaver/fabric/notebooks.py +201 -0
  77. weaver/fabric/onelake.py +263 -0
  78. weaver/fabric/resolution.py +344 -0
  79. weaver/fabric/resources.py +245 -0
  80. weaver/fabric/session.py +148 -0
  81. weaver/fabric/shortcuts.py +120 -0
  82. weaver/fabric/sql.py +118 -0
  83. weaver/fabric/store.py +198 -0
  84. weaver/initialise.py +209 -0
  85. weaver/lakehouse.py +386 -0
  86. weaver/load.py +474 -0
  87. weaver/load_execution.py +483 -0
  88. weaver/load_plan.py +912 -0
  89. weaver/load_report.py +330 -0
  90. weaver/load_resolution.py +386 -0
  91. weaver/locations.py +164 -0
  92. weaver/objects.py +392 -0
  93. weaver/operations.py +757 -0
  94. weaver/physical_wipe.py +369 -0
  95. weaver/push.py +76 -0
  96. weaver/resolution.py +292 -0
  97. weaver/runtime/__init__.py +30 -0
  98. weaver/runtime/folder_load.py +402 -0
  99. weaver/runtime/load_contract.py +245 -0
  100. weaver/runtime/load_result.py +104 -0
  101. weaver/runtime/spark_load.py +152 -0
  102. weaver/runtime/table_load.py +497 -0
  103. weaver/spark/__init__.py +49 -0
  104. weaver/spark/catalogue.py +245 -0
  105. weaver/spark/destination.py +195 -0
  106. weaver/spark/session.py +84 -0
  107. weaver/spark/tokens.py +138 -0
  108. weaver/sql/__init__.py +40 -0
  109. weaver/sql/authentication.py +38 -0
  110. weaver/sql/connection.py +90 -0
  111. weaver/sql/errors.py +25 -0
  112. weaver/sql/execution.py +123 -0
  113. weaver/sql/pool.py +174 -0
  114. weaver/sql/wipe.py +156 -0
  115. weaver/store.py +209 -0
  116. weaver/targets.py +257 -0
  117. weaver/task_logging.py +215 -0
  118. weaver/unbind.py +74 -0
  119. weaver/workspaces.py +175 -0
  120. weaver_cli/__init__.py +12 -0
  121. weaver_cli/__main__.py +7 -0
  122. weaver_cli/main.py +626 -0
  123. weaverstack-0.1.1.dist-info/METADATA +113 -0
  124. weaverstack-0.1.1.dist-info/RECORD +127 -0
  125. weaverstack-0.1.1.dist-info/WHEEL +4 -0
  126. weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
  127. weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,186 @@
1
+ """What a build intends each target to look like afterwards.
2
+
3
+ A plan says what will *run*. This says what it will *mean*: for each bound
4
+ target, the objects a build adds and the objects it removes. The two are written
5
+ side by side, deliberately, and a test holds them to each other.
6
+
7
+ **Why declare it rather than derive it.** The effect of an action could be
8
+ inferred — ``prune_schema`` removes a schema, ``write_file`` adds a path — but
9
+ that inference is a model of what executors do, living somewhere no executor can
10
+ correct. It drifts silently. Declaring the effect where the action is rendered
11
+ keeps the two statements next to each other in one function, which is where a
12
+ disagreement is cheapest to notice.
13
+
14
+ **Why that is not self-certification.** A summary the planner writes about its
15
+ own plan proves nothing on its own. What makes it load-bearing is
16
+ :data:`action_id`: every physical action must be named by exactly one change and
17
+ every change must name a real action, so the two cannot fall out of step without
18
+ a test failing. Adding an artefact type means emitting an action *and* a change;
19
+ forget either and the bijection breaks.
20
+
21
+ That is also why the identity lives here rather than on the action. A prune
22
+ action deliberately carries no ``resource_node_id`` — a pruned object has no node
23
+ in the repository, which is precisely why it is being pruned — so the thing a
24
+ prune removes has nowhere else to be written down. Here it has somewhere.
25
+
26
+ Applying these to a :class:`~weaver.build_bundle.prune.TargetInventory` gives the
27
+ state a build is aiming at, which is what lets "a build converges on what the
28
+ source declares" be asserted without installing anything.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ from dataclasses import dataclass
34
+ from typing import Any, Iterable, Mapping
35
+
36
+ from ..errors import BuildError
37
+
38
+ #: What a change does to the target.
39
+ ADD = "add"
40
+ REMOVE = "remove"
41
+ EFFECTS = (ADD, REMOVE)
42
+
43
+ #: What is being changed, in the vocabulary a ``TargetInventory`` holds. Every
44
+ #: name here is one of that class's collections, because the whole point is to
45
+ #: land on it — a kind with nowhere to go would be a change nothing could apply.
46
+ SCHEMA = "schema"
47
+ TABLE = "table"
48
+ VIEW = "view"
49
+ FOLDER = "folder"
50
+ FOLDER_SCHEMA = "folder_schema"
51
+ FILE = "file"
52
+ STORED_PROCEDURE = "stored_procedure"
53
+ OBJECT_KINDS = (SCHEMA, TABLE, VIEW, FOLDER, FOLDER_SCHEMA, FILE, STORED_PROCEDURE)
54
+
55
+ #: Which inventory collection each kind lives in.
56
+ _COLLECTION = {
57
+ SCHEMA: "schemas",
58
+ TABLE: "tables",
59
+ VIEW: "views",
60
+ FOLDER: "folders",
61
+ FOLDER_SCHEMA: "folder_schemas",
62
+ FILE: "files",
63
+ STORED_PROCEDURE: "procedures",
64
+ }
65
+
66
+
67
+ @dataclass(frozen=True, order=True)
68
+ class TargetChange:
69
+ """One object a build will add to, or remove from, one target.
70
+
71
+ ``name`` is spelled exactly as the inventory spells it — ``DWG.Customer``,
72
+ ``_/Load/lib/dates.py`` — because it is compared against a real read. A
73
+ change whose name did not match what a target reports would apply cleanly and
74
+ describe nothing.
75
+ """
76
+
77
+ effect: str
78
+ object_kind: str
79
+ name: str
80
+ #: The action that brings this about. What makes the summary checkable.
81
+ action_id: str
82
+
83
+ def __post_init__(self) -> None:
84
+ if self.effect not in EFFECTS:
85
+ raise BuildError(
86
+ f"change effect must be one of {', '.join(EFFECTS)}, got {self.effect!r}"
87
+ )
88
+ if self.object_kind not in OBJECT_KINDS:
89
+ raise BuildError(
90
+ f"change object kind must be one of {', '.join(OBJECT_KINDS)}, "
91
+ f"got {self.object_kind!r}"
92
+ )
93
+ if not self.name or not self.action_id:
94
+ raise BuildError("a change needs both a name and the action producing it")
95
+
96
+ def to_mapping(self) -> dict[str, Any]:
97
+ return {
98
+ "effect": self.effect,
99
+ "object_kind": self.object_kind,
100
+ "name": self.name,
101
+ "action_id": self.action_id,
102
+ }
103
+
104
+ @classmethod
105
+ def from_mapping(cls, mapping: Mapping[str, Any]) -> "TargetChange":
106
+ return cls(
107
+ effect=str(mapping["effect"]),
108
+ object_kind=str(mapping["object_kind"]),
109
+ name=str(mapping["name"]),
110
+ action_id=str(mapping["action_id"]),
111
+ )
112
+
113
+
114
+ def added(object_kind: str, name: str, action_id: str) -> TargetChange:
115
+ return TargetChange(ADD, object_kind, name, action_id)
116
+
117
+
118
+ def removed(object_kind: str, name: str, action_id: str) -> TargetChange:
119
+ return TargetChange(REMOVE, object_kind, name, action_id)
120
+
121
+
122
+ def merge(
123
+ *sections: Mapping[str, Iterable[TargetChange]]
124
+ ) -> dict[str, tuple[TargetChange, ...]]:
125
+ """Fold several target-keyed change sets into one, preserving order."""
126
+
127
+ merged: dict[str, list[TargetChange]] = {}
128
+ for section in sections:
129
+ for target_id, changes in section.items():
130
+ merged.setdefault(target_id, []).extend(changes)
131
+ return {target_id: tuple(changes) for target_id, changes in merged.items()}
132
+
133
+
134
+ def apply_to(inventory, changes: Iterable[TargetChange]):
135
+ """The inventory a target would hold once these changes have been made.
136
+
137
+ Pure, and returns a new inventory: the point is to compare a *predicted*
138
+ state against a declared one, and mutating the input would make the two
139
+ comparable only once.
140
+
141
+ Folder schemas are derived rather than declared where they can be. Adding
142
+ ``Raw.CustomerCsv`` implies the ``Raw`` area exists, and a target reports it
143
+ that way; requiring a build to say so separately would be a second thing to
144
+ keep in step for no gain. Removing a whole area is declared, because that is
145
+ a decision rather than a consequence.
146
+ """
147
+
148
+ from dataclasses import replace
149
+
150
+ collections = {
151
+ field: list(getattr(inventory, field)) for field in set(_COLLECTION.values())
152
+ }
153
+ for change in changes:
154
+ collection = collections[_COLLECTION[change.object_kind]]
155
+ folded = {value.casefold() for value in collection}
156
+ if change.effect == ADD:
157
+ if change.name.casefold() not in folded:
158
+ collection.append(change.name)
159
+ else:
160
+ collection[:] = [
161
+ value
162
+ for value in collection
163
+ if value.casefold() != change.name.casefold()
164
+ ]
165
+ if change.object_kind == FOLDER_SCHEMA:
166
+ # Removing an area takes what is inside it with it, exactly as
167
+ # deleting the directory does.
168
+ prefix = f"{change.name.casefold()}."
169
+ collections["folders"] = [
170
+ value
171
+ for value in collections["folders"]
172
+ if not value.casefold().startswith(prefix)
173
+ ]
174
+
175
+ implied = {value.split(".", 1)[0] for value in collections["folders"]}
176
+ for area in sorted(implied):
177
+ if area.casefold() not in {v.casefold() for v in collections["folder_schemas"]}:
178
+ collections["folder_schemas"].append(area)
179
+
180
+ return replace(
181
+ inventory,
182
+ **{
183
+ field: tuple(sorted(values, key=str.casefold))
184
+ for field, values in collections.items()
185
+ },
186
+ )
@@ -0,0 +1,83 @@
1
+ """The refresh that closes one Lakehouse item's physical work.
2
+
3
+ A Fabric Lakehouse presents its Delta tables twice: natively to Spark, and
4
+ through a SQL analytics endpoint whose metadata is synchronised behind the
5
+ mutation rather than with it. Everything that reads a Lakehouse *as SQL* reads
6
+ that endpoint — a Warehouse view over another item, a report, a downstream
7
+ shortcut — so a build that created a table and immediately built a dependent view
8
+ over it could and did see the previous shape.
9
+
10
+ The refresh therefore sits at the item boundary, not in a tail after every item:
11
+ it is the completion barrier for the item that mutated Delta, and it has to be
12
+ behind that item and ahead of anything in a later item layer.
13
+
14
+ It is planned host-independently, exactly like the rest of the bundle. The
15
+ emulator has no SQL analytics endpoint at all, and the executor says so and skips
16
+ rather than inventing a local equivalent that would keep no promise.
17
+
18
+ The *placement* is this module's business; the refresh itself belongs to
19
+ :mod:`weaver.build_bundle.executors.sql_endpoint_refresh`. An earlier design put
20
+ one refresh in a global tail after all physical work, which is correct for a
21
+ single item and wrong the moment a second item reads the first: the consumer's
22
+ Warehouse view would be created against endpoint metadata that had not caught up.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from typing import Iterable
28
+
29
+ from ..declaration.model import WeaverItemId
30
+ from .models import CREATE_ALIAS, REFRESH_SQL_ENDPOINT, BuildAction, BuildBatch
31
+ from .physical import DELTA_MUTATING_KINDS
32
+ from .stages import REFRESH, PlannedStage
33
+ from .targets import BoundTarget, WAREHOUSE_TARGET
34
+
35
+ #: Everything that leaves a Lakehouse's endpoint metadata stale. Alias creation
36
+ #: is here because a OneLake shortcut *is* a new table in the destination.
37
+ _MUTATING = DELTA_MUTATING_KINDS | {CREATE_ALIAS}
38
+
39
+
40
+ def item_refresh_stage(
41
+ stages: Iterable[PlannedStage],
42
+ *,
43
+ item: WeaverItemId,
44
+ target: BoundTarget,
45
+ ) -> PlannedStage | None:
46
+ """One refresh for this item, when its planned work mutated Delta.
47
+
48
+ A Warehouse item has no endpoint of its own to refresh — it *is* reached over
49
+ SQL — and an item whose only work was a folder or a schema has changed
50
+ nothing the endpoint describes.
51
+ """
52
+
53
+ if target.kind == WAREHOUSE_TARGET:
54
+ return None
55
+ if not any(
56
+ action.kind in _MUTATING
57
+ for stage in stages
58
+ for batch in stage.batches
59
+ for action in batch.actions
60
+ ):
61
+ return None
62
+ slug = str(item).replace("/", "--").replace(" ", "-")
63
+ return PlannedStage(
64
+ phase=REFRESH,
65
+ slug="refresh-endpoints",
66
+ description="refresh mutated Lakehouse SQL endpoints",
67
+ batches=(
68
+ BuildBatch(
69
+ id=f"refresh-endpoint-{slug}",
70
+ target_id=target.id,
71
+ actions=(
72
+ BuildAction(
73
+ id=f"refresh-sql-endpoint-{slug}",
74
+ kind=REFRESH_SQL_ENDPOINT,
75
+ resource_node_id=None,
76
+ executor="sql_endpoint_refresh",
77
+ payload=None,
78
+ payload_sha256=None,
79
+ ),
80
+ ),
81
+ ),
82
+ ),
83
+ )
@@ -0,0 +1,69 @@
1
+ """Executor dispatch for build actions.
2
+
3
+ ``spark_sql`` runs one create or frozen ``DROP``; ``spark_sql_batch`` runs an
4
+ ordered catalogue payload as one reported action. ``spark_schema`` makes one
5
+ schema, ``spark_table`` completes a table whose shape only the session knows,
6
+ and ``folder`` makes or removes a directory. ``tsql`` is the Warehouse SQL path.
7
+ ``alias`` points one Lakehouse name at another item's object, and ``sql_endpoint``
8
+ syncs a Lakehouse's SQL analytics endpoint.
9
+
10
+ ``load_file`` closes an item: it writes one file of the deployed runtime tree, or
11
+ removes one the source has stopped claiming. Generated load procedures need no
12
+ executor of their own — a create-or-alter is T-SQL, and ``tsql`` runs the payload
13
+ it is given without inspecting what the payload builds.
14
+
15
+ There is no prune executor — a build freezes its drops as payloads, so the
16
+ installer never enumerates the target.
17
+
18
+ Every Spark executor addresses the destination its batch names, and none of them
19
+ relies on what the session is attached to.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from .alias import AliasExecutor
25
+ from .base import ActionExecutor, InstallationContext, ResolvedTarget, SkippedExecution
26
+ from .folder import FolderExecutor
27
+ from .load_file import LoadFileExecutor
28
+ from .spark_schema import SparkSchemaExecutor
29
+ from .spark_sql import SparkSqlExecutor
30
+ from .spark_sql_batch import SparkSqlBatchExecutor
31
+ from .spark_table import SparkTableExecutor
32
+ from .sql_endpoint_refresh import SqlEndpointRefreshExecutor
33
+ from .tsql import TSqlBatchExecutor, TSqlExecutor
34
+
35
+
36
+ def default_executors() -> dict[str, ActionExecutor]:
37
+ """The executor registry, by name — the names actions carry."""
38
+
39
+ return {
40
+ SparkSqlExecutor.name: SparkSqlExecutor(),
41
+ SparkSqlBatchExecutor.name: SparkSqlBatchExecutor(),
42
+ SparkSchemaExecutor.name: SparkSchemaExecutor(),
43
+ SparkTableExecutor.name: SparkTableExecutor(),
44
+ FolderExecutor.name: FolderExecutor(),
45
+ LoadFileExecutor.name: LoadFileExecutor(),
46
+ TSqlExecutor.name: TSqlExecutor(),
47
+ TSqlBatchExecutor.name: TSqlBatchExecutor(),
48
+ AliasExecutor.name: AliasExecutor(),
49
+ SqlEndpointRefreshExecutor.name: SqlEndpointRefreshExecutor(),
50
+ }
51
+
52
+
53
+ __all__ = [
54
+ "ActionExecutor",
55
+ "InstallationContext",
56
+ "ResolvedTarget",
57
+ "AliasExecutor",
58
+ "SkippedExecution",
59
+ "SparkSchemaExecutor",
60
+ "SparkSqlExecutor",
61
+ "SparkSqlBatchExecutor",
62
+ "SparkTableExecutor",
63
+ "SqlEndpointRefreshExecutor",
64
+ "FolderExecutor",
65
+ "LoadFileExecutor",
66
+ "TSqlExecutor",
67
+ "TSqlBatchExecutor",
68
+ "default_executors",
69
+ ]
@@ -0,0 +1,202 @@
1
+ """Materialising one Lakehouse alias — a pointer, in whatever form the host has.
2
+
3
+ The payload names two addresses and nothing else: this item's ``Schema.Object``,
4
+ and the object in another item it stands for. Both are resolved through the
5
+ environment, the same way every other action's destination is, so the bundle
6
+ carries no path from the machine that wrote it.
7
+
8
+ The *form* the pointer takes is transport:
9
+
10
+ ``Fabric``
11
+ a OneLake shortcut in the destination Lakehouse, created through the
12
+ workspace's own API.
13
+ ``the local emulator``
14
+ a filesystem link beside the destination's other tables, which is what makes
15
+ the emulator's ``Tables/`` area keep mirroring what a shortcut looks like in
16
+ OneLake — plus the catalogue registration Fabric performs for itself, because
17
+ a link no statement could name would not be an alias at all.
18
+
19
+ That is the same split :mod:`weaver.build_bundle.executors.spark_schema`
20
+ documents: which alias, over what, is settled and in the manifest; how a name is
21
+ made to point somewhere is the environment's business. An alias holds no data, so
22
+ an existing one is replaced rather than treated as an unexpected collision — a
23
+ build has to be able to run twice.
24
+
25
+ **The action is not finished until the alias can be read.** Fabric creates a
26
+ shortcut synchronously and *discovers* it asynchronously, so the API call
27
+ returning is not the same thing as the alias existing: for a few seconds the
28
+ Lakehouse reports the name as "neither a view nor a table". An action that
29
+ reported success there would make the barrier the plan puts around it a lie, and
30
+ the failure would surface in the next item's DDL — which is exactly where it did
31
+ surface, before this waited.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ import json
37
+ import time
38
+ from typing import Any
39
+
40
+ from ...errors import InstallError
41
+ from ...targets import DeltaTarget, FolderTarget
42
+ from ..models import BuildAction
43
+ from .base import InstallationContext, ResolvedTarget
44
+ from .spark_case import exact_identifier_case
45
+
46
+ FILES_AREA = "Files"
47
+
48
+ #: How long a freshly created shortcut may take to become addressable, and how
49
+ #: often to ask. Discovery normally takes seconds; the bound exists so a
50
+ #: never-appearing alias fails naming itself rather than as an obscure error in
51
+ #: whatever statement reads it next.
52
+ ADDRESSABLE_TIMEOUT = 300.0
53
+ ADDRESSABLE_POLL_INTERVAL = 5.0
54
+
55
+
56
+ class AliasExecutor:
57
+ name = "alias"
58
+
59
+ def execute(
60
+ self,
61
+ action: BuildAction,
62
+ payload: bytes | None,
63
+ context: InstallationContext,
64
+ ) -> dict[str, Any] | None:
65
+ if payload is None:
66
+ raise InstallError(f"alias action {action.id!r} has no payload")
67
+ frozen = json.loads(payload.decode("utf-8"))["aliases"]
68
+ if not frozen:
69
+ return {"aliases": []}
70
+
71
+ shortcut = getattr(context.resolver, "create_onelake_shortcut", None)
72
+ link = getattr(context.store, "link", None)
73
+ if shortcut is None and link is None:
74
+ raise InstallError(
75
+ f"alias action {action.id!r} cannot be materialised here: this "
76
+ "environment offers neither a OneLake shortcut nor a store link"
77
+ )
78
+
79
+ made = []
80
+ for each in frozen:
81
+ source = context.resolved(each["source_target_id"])
82
+ if shortcut is not None:
83
+ made.append(self._shortcut(shortcut, each, context, source))
84
+ else:
85
+ made.append(self._link(link, each, context, source))
86
+
87
+ details: dict[str, Any] = {"aliases": made}
88
+ # Every shortcut is created before anything waits, so the cost is one
89
+ # discovery window rather than one per alias.
90
+ if shortcut is not None and context.spark is not None:
91
+ waited = self._await_addressable(context, frozen)
92
+ if waited is not None:
93
+ details["addressable_after_seconds"] = waited
94
+ return details
95
+
96
+ def _shortcut(self, shortcut, frozen: dict, context, source) -> dict:
97
+ made = shortcut(
98
+ context.target.lakehouse,
99
+ path=f"{frozen['area']}/{frozen['schema']}",
100
+ name=frozen["object"],
101
+ source=source.lakehouse,
102
+ source_path=(
103
+ f"{frozen['source_area']}/{frozen['source_schema']}"
104
+ f"/{frozen['source_object']}"
105
+ ),
106
+ )
107
+ return {"alias": frozen["alias"], "source": frozen["source"], **(made or {})}
108
+
109
+ def _link(self, link, frozen: dict, context, source) -> dict:
110
+ destination = _location(context.target, frozen, context, source=False)
111
+ producer = _location(source, frozen, context, source=True)
112
+ if not context.store.exists(producer):
113
+ raise InstallError(
114
+ f"alias {frozen['alias']} has no source to point at: "
115
+ f"{producer.value} does not exist"
116
+ )
117
+ if context.store.exists(destination):
118
+ context.store.delete(destination, recursive=True)
119
+ link(producer, destination)
120
+ made = {
121
+ "alias": frozen["alias"],
122
+ "source": frozen["source"],
123
+ "linked": destination.value,
124
+ "to": producer.value,
125
+ }
126
+ if frozen["area"] != FILES_AREA and context.spark is not None:
127
+ # Fabric discovers a shortcut under Tables/ and the table appears in
128
+ # the catalogue. Local Spark discovers nothing, so the emulator names
129
+ # it — otherwise the link would exist and no statement could reach it.
130
+ catalogue = context.catalogue
131
+ with exact_identifier_case(
132
+ context.spark,
133
+ enabled=catalogue.destination.preserve_table_identifier_case,
134
+ ):
135
+ made["registered"] = catalogue.register_external_table(
136
+ frozen["schema"], frozen["object"], destination.value
137
+ )
138
+ return made
139
+
140
+ def _await_addressable(self, context: InstallationContext, frozen: list) -> float | None:
141
+ """Wait until every table alias just created can actually be read.
142
+
143
+ The probe is a read, not a catalogue lookup, because the catalogue is the
144
+ thing that is briefly wrong: Fabric reports the shortcut's metadata
145
+ location while still refusing it as a relation. Only a read that succeeds
146
+ proves the alias is usable, and a read is what the next action does.
147
+
148
+ All of them are waited on together, so several aliases cost one discovery
149
+ window instead of one each.
150
+ """
151
+
152
+ catalogue = context.catalogue
153
+ pending = {
154
+ each["alias"]: catalogue.qualify(each["schema"], each["object"])
155
+ for each in frozen
156
+ if each["area"] != FILES_AREA
157
+ }
158
+ if not pending:
159
+ return None
160
+
161
+ started = time.monotonic()
162
+ deadline = started + ADDRESSABLE_TIMEOUT
163
+ failure: Exception | None = None
164
+ while pending:
165
+ for alias, qualified in list(pending.items()):
166
+ try:
167
+ with exact_identifier_case(
168
+ context.spark,
169
+ enabled=catalogue.destination.preserve_table_identifier_case,
170
+ ):
171
+ context.spark.sql(f"SELECT * FROM {qualified} LIMIT 0").collect()
172
+ del pending[alias]
173
+ except Exception as exc: # not discovered yet — or never will be
174
+ failure = exc
175
+ if not pending:
176
+ break
177
+ if time.monotonic() >= deadline:
178
+ raise InstallError(
179
+ f"alias(es) {', '.join(sorted(pending))} were created but did "
180
+ f"not become readable within {int(ADDRESSABLE_TIMEOUT)}s: {failure}"
181
+ ) from failure
182
+ time.sleep(ADDRESSABLE_POLL_INTERVAL)
183
+ return round(time.monotonic() - started, 1)
184
+
185
+
186
+ def _location(
187
+ target: ResolvedTarget,
188
+ frozen: dict,
189
+ context: InstallationContext,
190
+ *,
191
+ source: bool,
192
+ ):
193
+ area = frozen["source_area"] if source else frozen["area"]
194
+ schema = frozen["source_schema"] if source else frozen["schema"]
195
+ name = frozen["source_object"] if source else frozen["object"]
196
+ if area == FILES_AREA:
197
+ return context.resolver.folder_object(
198
+ FolderTarget(lakehouse=target.lakehouse), schema, name
199
+ )
200
+ return context.resolver.delta_table(
201
+ DeltaTarget(lakehouse=target.lakehouse), schema, name
202
+ )
@@ -0,0 +1,132 @@
1
+ """The executor seam — dispatch only, no planning.
2
+
3
+ An executor runs one action's payload against one resolved target and returns
4
+ optional structured details, or raises. It never reads the repository, resolves
5
+ a dependency or selects a target: those decisions are all in the bundle already.
6
+ The installer owns timing, status and reporting; an executor owns the work.
7
+
8
+ The context carries runtime services — a Spark session, the resolver and store —
9
+ plus the one target the current batch is bound to. It carries no planning input,
10
+ and no way back to the repository: everything an action needs is its payload.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass, field
16
+ from typing import Any, Mapping, Protocol
17
+
18
+ from ...errors import InstallError
19
+ from ...locations import LakehouseSparkLocation
20
+ from ...spark import SparkCatalogue, SparkDestination
21
+ from ...store import Store
22
+ from ...targets import ItemRef
23
+ from ..models import BuildAction
24
+ from ..targets import BoundTarget
25
+
26
+
27
+ @dataclass(frozen=True)
28
+ class ResolvedTarget:
29
+ """A manifest target resolved to what the executor addresses.
30
+
31
+ ``lakehouse`` is the logical item the bundle named. The other two are that
32
+ item resolved into the two things Spark needs to reach it, and they are two
33
+ because Fabric answers them separately:
34
+
35
+ ``location``
36
+ the physical roots — where the bytes are. An ``abfss://`` URL on Fabric, a
37
+ directory locally.
38
+ ``destination``
39
+ the catalogue name — what a statement calls it. Fabric's four-part
40
+ ``workspace.lakehouse.schema.object``; locally the folded database name.
41
+
42
+ Both are needed, and neither substitutes for the other: a folder is created at
43
+ a path and has no catalogue name, while a view exists only as a name and has no
44
+ path of its own.
45
+
46
+ Resolution happens here, once per target, rather than in each executor. An
47
+ executor that derived either for itself would be re-deciding where an action
48
+ lands, which is a planning decision it is not allowed to make. It is also what
49
+ lets one session build several destinations, and write the catalogue to a
50
+ different one again, without ever switching what the session is attached to.
51
+
52
+ Both are None for a Warehouse target, which is reached over TDS and has
53
+ neither.
54
+ """
55
+
56
+ bound: BoundTarget
57
+ lakehouse: ItemRef
58
+ location: LakehouseSparkLocation | None = None
59
+ destination: SparkDestination | None = None
60
+
61
+
62
+ @dataclass(frozen=True)
63
+ class InstallationContext:
64
+ """Runtime services and the one target the current batch is bound to.
65
+
66
+ ``spark`` runs Lakehouse work; ``sql`` runs Warehouse (T-SQL) work. A batch
67
+ names one target, so only the capability its actions need has to be present.
68
+
69
+ ``targets`` holds every target the plan declared, already resolved. It exists
70
+ for the one action that legitimately spans two of them — an alias, which
71
+ points a name in ``target`` at an object in another — and it carries resolved
72
+ targets rather than ids so that a second destination is addressed exactly as
73
+ the batch's own is, and never derived by an executor.
74
+ """
75
+
76
+ spark: Any
77
+ resolver: Any
78
+ store: Store
79
+ target: ResolvedTarget
80
+ sql: Any = None
81
+ targets: Mapping[str, ResolvedTarget] = field(default_factory=dict)
82
+ #: This installation's publication instant, resolved into ``{{epoch}}``. One
83
+ #: value for the whole run, so every Registry row a build writes carries the
84
+ #: same one and two rows can be ordered against each other. It is not a
85
+ #: destination's business, which is why it lives here rather than being
86
+ #: resolved with the object tokens.
87
+ epoch: str | None = None
88
+
89
+ def resolved(self, target_id: str) -> ResolvedTarget:
90
+ """Another target this plan declared, by the id an action names."""
91
+
92
+ found = self.targets.get(target_id)
93
+ if found is None:
94
+ raise InstallError(
95
+ f"action names target {target_id!r}, which this plan does not declare"
96
+ )
97
+ return found
98
+
99
+ @property
100
+ def catalogue(self) -> SparkCatalogue:
101
+ """Catalogue operations against *this batch's* destination.
102
+
103
+ Built per access rather than stored, so the context stays a frozen record
104
+ of what was resolved. Failing here — rather than falling back to the
105
+ session's own catalogue — is the point: an action with nowhere to go must
106
+ stop, not land somewhere plausible (how-does-build-work §4).
107
+ """
108
+
109
+ if self.target.destination is None:
110
+ raise InstallError(
111
+ f"target {self.target.bound.id!r} resolved to no Spark destination, "
112
+ "so a statement naming an object has nowhere to run"
113
+ )
114
+ return SparkCatalogue(self.spark, self.target.destination)
115
+
116
+
117
+ @dataclass(frozen=True)
118
+ class SkippedExecution:
119
+ """An executor's explicit, non-failing decision not to run on this host."""
120
+
121
+ details: dict[str, Any] | None = None
122
+
123
+
124
+ class ActionExecutor(Protocol):
125
+ name: str
126
+
127
+ def execute(
128
+ self,
129
+ action: BuildAction,
130
+ payload: bytes | None,
131
+ context: InstallationContext,
132
+ ) -> dict[str, Any] | SkippedExecution | None: ...