weaverstack 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weaver/__init__.py +59 -0
- weaver/build_bundle/__init__.py +109 -0
- weaver/build_bundle/aliases.py +325 -0
- weaver/build_bundle/bundle.py +359 -0
- weaver/build_bundle/catalogue_actions.py +275 -0
- weaver/build_bundle/changes.py +186 -0
- weaver/build_bundle/endpoints.py +83 -0
- weaver/build_bundle/executors/__init__.py +69 -0
- weaver/build_bundle/executors/alias.py +202 -0
- weaver/build_bundle/executors/base.py +132 -0
- weaver/build_bundle/executors/folder.py +71 -0
- weaver/build_bundle/executors/load_file.py +205 -0
- weaver/build_bundle/executors/spark_case.py +26 -0
- weaver/build_bundle/executors/spark_schema.py +60 -0
- weaver/build_bundle/executors/spark_sql.py +59 -0
- weaver/build_bundle/executors/spark_sql_batch.py +57 -0
- weaver/build_bundle/executors/spark_table.py +213 -0
- weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
- weaver/build_bundle/executors/tsql.py +81 -0
- weaver/build_bundle/incremental.py +288 -0
- weaver/build_bundle/installer.py +384 -0
- weaver/build_bundle/models.py +288 -0
- weaver/build_bundle/payloads.py +34 -0
- weaver/build_bundle/physical.py +625 -0
- weaver/build_bundle/planner.py +389 -0
- weaver/build_bundle/prune.py +620 -0
- weaver/build_bundle/report.py +108 -0
- weaver/build_bundle/stages.py +196 -0
- weaver/build_bundle/targets.py +272 -0
- weaver/build_bundle/workflow.py +585 -0
- weaver/catalogue/__init__.py +73 -0
- weaver/catalogue/builtin.py +238 -0
- weaver/catalogue/claims.py +121 -0
- weaver/catalogue/projection.py +437 -0
- weaver/catalogue/reader.py +152 -0
- weaver/catalogue/reconcile.py +231 -0
- weaver/catalogue/render.py +410 -0
- weaver/catalogue/state.py +660 -0
- weaver/catalogue/tables.py +648 -0
- weaver/config.py +178 -0
- weaver/declaration/__init__.py +171 -0
- weaver/declaration/columns.py +223 -0
- weaver/declaration/ddl.py +266 -0
- weaver/declaration/dependencies.py +544 -0
- weaver/declaration/graph.py +240 -0
- weaver/declaration/item_dependencies.py +292 -0
- weaver/declaration/load.py +191 -0
- weaver/declaration/metadata.py +1405 -0
- weaver/declaration/model.py +448 -0
- weaver/declaration/references.py +294 -0
- weaver/declaration/repository.py +959 -0
- weaver/declaration/schemas.py +135 -0
- weaver/declaration/source.py +674 -0
- weaver/declaration/spark_load.py +759 -0
- weaver/declaration/sql_shaping.py +591 -0
- weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
- weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
- weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
- weaver/declaration/templates/load/column_metadata.sql +40 -0
- weaver/declaration/templates/load/full_replace_body.sql +21 -0
- weaver/declaration/templates/load/install_load_procedure.sql +27 -0
- weaver/declaration/templates/load/load_procedure.sql +48 -0
- weaver/declaration/templates/load/primary_key_body.sql +113 -0
- weaver/declaration/tsql_ddl.py +468 -0
- weaver/declaration/tsql_load.py +417 -0
- weaver/declaration/warehouse_type_mapping.yml +93 -0
- weaver/diagnostics.py +247 -0
- weaver/errors.py +61 -0
- weaver/etl.py +469 -0
- weaver/fabric/__init__.py +107 -0
- weaver/fabric/auth.py +137 -0
- weaver/fabric/capacity.py +143 -0
- weaver/fabric/client.py +147 -0
- weaver/fabric/environment.py +460 -0
- weaver/fabric/livy.py +478 -0
- weaver/fabric/notebooks.py +201 -0
- weaver/fabric/onelake.py +263 -0
- weaver/fabric/resolution.py +344 -0
- weaver/fabric/resources.py +245 -0
- weaver/fabric/session.py +148 -0
- weaver/fabric/shortcuts.py +120 -0
- weaver/fabric/sql.py +118 -0
- weaver/fabric/store.py +198 -0
- weaver/initialise.py +209 -0
- weaver/lakehouse.py +386 -0
- weaver/load.py +474 -0
- weaver/load_execution.py +483 -0
- weaver/load_plan.py +912 -0
- weaver/load_report.py +330 -0
- weaver/load_resolution.py +386 -0
- weaver/locations.py +164 -0
- weaver/objects.py +392 -0
- weaver/operations.py +757 -0
- weaver/physical_wipe.py +369 -0
- weaver/push.py +76 -0
- weaver/resolution.py +292 -0
- weaver/runtime/__init__.py +30 -0
- weaver/runtime/folder_load.py +402 -0
- weaver/runtime/load_contract.py +245 -0
- weaver/runtime/load_result.py +104 -0
- weaver/runtime/spark_load.py +152 -0
- weaver/runtime/table_load.py +497 -0
- weaver/spark/__init__.py +49 -0
- weaver/spark/catalogue.py +245 -0
- weaver/spark/destination.py +195 -0
- weaver/spark/session.py +84 -0
- weaver/spark/tokens.py +138 -0
- weaver/sql/__init__.py +40 -0
- weaver/sql/authentication.py +38 -0
- weaver/sql/connection.py +90 -0
- weaver/sql/errors.py +25 -0
- weaver/sql/execution.py +123 -0
- weaver/sql/pool.py +174 -0
- weaver/sql/wipe.py +156 -0
- weaver/store.py +209 -0
- weaver/targets.py +257 -0
- weaver/task_logging.py +215 -0
- weaver/unbind.py +74 -0
- weaver/workspaces.py +175 -0
- weaver_cli/__init__.py +12 -0
- weaver_cli/__main__.py +7 -0
- weaver_cli/main.py +626 -0
- weaverstack-0.1.1.dist-info/METADATA +113 -0
- weaverstack-0.1.1.dist-info/RECORD +127 -0
- weaverstack-0.1.1.dist-info/WHEEL +4 -0
- weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
- weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""What a build intends each target to look like afterwards.
|
|
2
|
+
|
|
3
|
+
A plan says what will *run*. This says what it will *mean*: for each bound
|
|
4
|
+
target, the objects a build adds and the objects it removes. The two are written
|
|
5
|
+
side by side, deliberately, and a test holds them to each other.
|
|
6
|
+
|
|
7
|
+
**Why declare it rather than derive it.** The effect of an action could be
|
|
8
|
+
inferred — ``prune_schema`` removes a schema, ``write_file`` adds a path — but
|
|
9
|
+
that inference is a model of what executors do, living somewhere no executor can
|
|
10
|
+
correct. It drifts silently. Declaring the effect where the action is rendered
|
|
11
|
+
keeps the two statements next to each other in one function, which is where a
|
|
12
|
+
disagreement is cheapest to notice.
|
|
13
|
+
|
|
14
|
+
**Why that is not self-certification.** A summary the planner writes about its
|
|
15
|
+
own plan proves nothing on its own. What makes it load-bearing is
|
|
16
|
+
:data:`action_id`: every physical action must be named by exactly one change and
|
|
17
|
+
every change must name a real action, so the two cannot fall out of step without
|
|
18
|
+
a test failing. Adding an artefact type means emitting an action *and* a change;
|
|
19
|
+
forget either and the bijection breaks.
|
|
20
|
+
|
|
21
|
+
That is also why the identity lives here rather than on the action. A prune
|
|
22
|
+
action deliberately carries no ``resource_node_id`` — a pruned object has no node
|
|
23
|
+
in the repository, which is precisely why it is being pruned — so the thing a
|
|
24
|
+
prune removes has nowhere else to be written down. Here it has somewhere.
|
|
25
|
+
|
|
26
|
+
Applying these to a :class:`~weaver.build_bundle.prune.TargetInventory` gives the
|
|
27
|
+
state a build is aiming at, which is what lets "a build converges on what the
|
|
28
|
+
source declares" be asserted without installing anything.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from typing import Any, Iterable, Mapping
|
|
35
|
+
|
|
36
|
+
from ..errors import BuildError
|
|
37
|
+
|
|
38
|
+
#: What a change does to the target.
|
|
39
|
+
ADD = "add"
|
|
40
|
+
REMOVE = "remove"
|
|
41
|
+
EFFECTS = (ADD, REMOVE)
|
|
42
|
+
|
|
43
|
+
#: What is being changed, in the vocabulary a ``TargetInventory`` holds. Every
|
|
44
|
+
#: name here is one of that class's collections, because the whole point is to
|
|
45
|
+
#: land on it — a kind with nowhere to go would be a change nothing could apply.
|
|
46
|
+
SCHEMA = "schema"
|
|
47
|
+
TABLE = "table"
|
|
48
|
+
VIEW = "view"
|
|
49
|
+
FOLDER = "folder"
|
|
50
|
+
FOLDER_SCHEMA = "folder_schema"
|
|
51
|
+
FILE = "file"
|
|
52
|
+
STORED_PROCEDURE = "stored_procedure"
|
|
53
|
+
OBJECT_KINDS = (SCHEMA, TABLE, VIEW, FOLDER, FOLDER_SCHEMA, FILE, STORED_PROCEDURE)
|
|
54
|
+
|
|
55
|
+
#: Which inventory collection each kind lives in.
|
|
56
|
+
_COLLECTION = {
|
|
57
|
+
SCHEMA: "schemas",
|
|
58
|
+
TABLE: "tables",
|
|
59
|
+
VIEW: "views",
|
|
60
|
+
FOLDER: "folders",
|
|
61
|
+
FOLDER_SCHEMA: "folder_schemas",
|
|
62
|
+
FILE: "files",
|
|
63
|
+
STORED_PROCEDURE: "procedures",
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True, order=True)
|
|
68
|
+
class TargetChange:
|
|
69
|
+
"""One object a build will add to, or remove from, one target.
|
|
70
|
+
|
|
71
|
+
``name`` is spelled exactly as the inventory spells it — ``DWG.Customer``,
|
|
72
|
+
``_/Load/lib/dates.py`` — because it is compared against a real read. A
|
|
73
|
+
change whose name did not match what a target reports would apply cleanly and
|
|
74
|
+
describe nothing.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
effect: str
|
|
78
|
+
object_kind: str
|
|
79
|
+
name: str
|
|
80
|
+
#: The action that brings this about. What makes the summary checkable.
|
|
81
|
+
action_id: str
|
|
82
|
+
|
|
83
|
+
def __post_init__(self) -> None:
|
|
84
|
+
if self.effect not in EFFECTS:
|
|
85
|
+
raise BuildError(
|
|
86
|
+
f"change effect must be one of {', '.join(EFFECTS)}, got {self.effect!r}"
|
|
87
|
+
)
|
|
88
|
+
if self.object_kind not in OBJECT_KINDS:
|
|
89
|
+
raise BuildError(
|
|
90
|
+
f"change object kind must be one of {', '.join(OBJECT_KINDS)}, "
|
|
91
|
+
f"got {self.object_kind!r}"
|
|
92
|
+
)
|
|
93
|
+
if not self.name or not self.action_id:
|
|
94
|
+
raise BuildError("a change needs both a name and the action producing it")
|
|
95
|
+
|
|
96
|
+
def to_mapping(self) -> dict[str, Any]:
|
|
97
|
+
return {
|
|
98
|
+
"effect": self.effect,
|
|
99
|
+
"object_kind": self.object_kind,
|
|
100
|
+
"name": self.name,
|
|
101
|
+
"action_id": self.action_id,
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def from_mapping(cls, mapping: Mapping[str, Any]) -> "TargetChange":
|
|
106
|
+
return cls(
|
|
107
|
+
effect=str(mapping["effect"]),
|
|
108
|
+
object_kind=str(mapping["object_kind"]),
|
|
109
|
+
name=str(mapping["name"]),
|
|
110
|
+
action_id=str(mapping["action_id"]),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def added(object_kind: str, name: str, action_id: str) -> TargetChange:
|
|
115
|
+
return TargetChange(ADD, object_kind, name, action_id)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def removed(object_kind: str, name: str, action_id: str) -> TargetChange:
|
|
119
|
+
return TargetChange(REMOVE, object_kind, name, action_id)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def merge(
|
|
123
|
+
*sections: Mapping[str, Iterable[TargetChange]]
|
|
124
|
+
) -> dict[str, tuple[TargetChange, ...]]:
|
|
125
|
+
"""Fold several target-keyed change sets into one, preserving order."""
|
|
126
|
+
|
|
127
|
+
merged: dict[str, list[TargetChange]] = {}
|
|
128
|
+
for section in sections:
|
|
129
|
+
for target_id, changes in section.items():
|
|
130
|
+
merged.setdefault(target_id, []).extend(changes)
|
|
131
|
+
return {target_id: tuple(changes) for target_id, changes in merged.items()}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def apply_to(inventory, changes: Iterable[TargetChange]):
|
|
135
|
+
"""The inventory a target would hold once these changes have been made.
|
|
136
|
+
|
|
137
|
+
Pure, and returns a new inventory: the point is to compare a *predicted*
|
|
138
|
+
state against a declared one, and mutating the input would make the two
|
|
139
|
+
comparable only once.
|
|
140
|
+
|
|
141
|
+
Folder schemas are derived rather than declared where they can be. Adding
|
|
142
|
+
``Raw.CustomerCsv`` implies the ``Raw`` area exists, and a target reports it
|
|
143
|
+
that way; requiring a build to say so separately would be a second thing to
|
|
144
|
+
keep in step for no gain. Removing a whole area is declared, because that is
|
|
145
|
+
a decision rather than a consequence.
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
from dataclasses import replace
|
|
149
|
+
|
|
150
|
+
collections = {
|
|
151
|
+
field: list(getattr(inventory, field)) for field in set(_COLLECTION.values())
|
|
152
|
+
}
|
|
153
|
+
for change in changes:
|
|
154
|
+
collection = collections[_COLLECTION[change.object_kind]]
|
|
155
|
+
folded = {value.casefold() for value in collection}
|
|
156
|
+
if change.effect == ADD:
|
|
157
|
+
if change.name.casefold() not in folded:
|
|
158
|
+
collection.append(change.name)
|
|
159
|
+
else:
|
|
160
|
+
collection[:] = [
|
|
161
|
+
value
|
|
162
|
+
for value in collection
|
|
163
|
+
if value.casefold() != change.name.casefold()
|
|
164
|
+
]
|
|
165
|
+
if change.object_kind == FOLDER_SCHEMA:
|
|
166
|
+
# Removing an area takes what is inside it with it, exactly as
|
|
167
|
+
# deleting the directory does.
|
|
168
|
+
prefix = f"{change.name.casefold()}."
|
|
169
|
+
collections["folders"] = [
|
|
170
|
+
value
|
|
171
|
+
for value in collections["folders"]
|
|
172
|
+
if not value.casefold().startswith(prefix)
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
implied = {value.split(".", 1)[0] for value in collections["folders"]}
|
|
176
|
+
for area in sorted(implied):
|
|
177
|
+
if area.casefold() not in {v.casefold() for v in collections["folder_schemas"]}:
|
|
178
|
+
collections["folder_schemas"].append(area)
|
|
179
|
+
|
|
180
|
+
return replace(
|
|
181
|
+
inventory,
|
|
182
|
+
**{
|
|
183
|
+
field: tuple(sorted(values, key=str.casefold))
|
|
184
|
+
for field, values in collections.items()
|
|
185
|
+
},
|
|
186
|
+
)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""The refresh that closes one Lakehouse item's physical work.
|
|
2
|
+
|
|
3
|
+
A Fabric Lakehouse presents its Delta tables twice: natively to Spark, and
|
|
4
|
+
through a SQL analytics endpoint whose metadata is synchronised behind the
|
|
5
|
+
mutation rather than with it. Everything that reads a Lakehouse *as SQL* reads
|
|
6
|
+
that endpoint — a Warehouse view over another item, a report, a downstream
|
|
7
|
+
shortcut — so a build that created a table and immediately built a dependent view
|
|
8
|
+
over it could and did see the previous shape.
|
|
9
|
+
|
|
10
|
+
The refresh therefore sits at the item boundary, not in a tail after every item:
|
|
11
|
+
it is the completion barrier for the item that mutated Delta, and it has to be
|
|
12
|
+
behind that item and ahead of anything in a later item layer.
|
|
13
|
+
|
|
14
|
+
It is planned host-independently, exactly like the rest of the bundle. The
|
|
15
|
+
emulator has no SQL analytics endpoint at all, and the executor says so and skips
|
|
16
|
+
rather than inventing a local equivalent that would keep no promise.
|
|
17
|
+
|
|
18
|
+
The *placement* is this module's business; the refresh itself belongs to
|
|
19
|
+
:mod:`weaver.build_bundle.executors.sql_endpoint_refresh`. An earlier design put
|
|
20
|
+
one refresh in a global tail after all physical work, which is correct for a
|
|
21
|
+
single item and wrong the moment a second item reads the first: the consumer's
|
|
22
|
+
Warehouse view would be created against endpoint metadata that had not caught up.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from typing import Iterable
|
|
28
|
+
|
|
29
|
+
from ..declaration.model import WeaverItemId
|
|
30
|
+
from .models import CREATE_ALIAS, REFRESH_SQL_ENDPOINT, BuildAction, BuildBatch
|
|
31
|
+
from .physical import DELTA_MUTATING_KINDS
|
|
32
|
+
from .stages import REFRESH, PlannedStage
|
|
33
|
+
from .targets import BoundTarget, WAREHOUSE_TARGET
|
|
34
|
+
|
|
35
|
+
#: Everything that leaves a Lakehouse's endpoint metadata stale. Alias creation
|
|
36
|
+
#: is here because a OneLake shortcut *is* a new table in the destination.
|
|
37
|
+
_MUTATING = DELTA_MUTATING_KINDS | {CREATE_ALIAS}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def item_refresh_stage(
|
|
41
|
+
stages: Iterable[PlannedStage],
|
|
42
|
+
*,
|
|
43
|
+
item: WeaverItemId,
|
|
44
|
+
target: BoundTarget,
|
|
45
|
+
) -> PlannedStage | None:
|
|
46
|
+
"""One refresh for this item, when its planned work mutated Delta.
|
|
47
|
+
|
|
48
|
+
A Warehouse item has no endpoint of its own to refresh — it *is* reached over
|
|
49
|
+
SQL — and an item whose only work was a folder or a schema has changed
|
|
50
|
+
nothing the endpoint describes.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
if target.kind == WAREHOUSE_TARGET:
|
|
54
|
+
return None
|
|
55
|
+
if not any(
|
|
56
|
+
action.kind in _MUTATING
|
|
57
|
+
for stage in stages
|
|
58
|
+
for batch in stage.batches
|
|
59
|
+
for action in batch.actions
|
|
60
|
+
):
|
|
61
|
+
return None
|
|
62
|
+
slug = str(item).replace("/", "--").replace(" ", "-")
|
|
63
|
+
return PlannedStage(
|
|
64
|
+
phase=REFRESH,
|
|
65
|
+
slug="refresh-endpoints",
|
|
66
|
+
description="refresh mutated Lakehouse SQL endpoints",
|
|
67
|
+
batches=(
|
|
68
|
+
BuildBatch(
|
|
69
|
+
id=f"refresh-endpoint-{slug}",
|
|
70
|
+
target_id=target.id,
|
|
71
|
+
actions=(
|
|
72
|
+
BuildAction(
|
|
73
|
+
id=f"refresh-sql-endpoint-{slug}",
|
|
74
|
+
kind=REFRESH_SQL_ENDPOINT,
|
|
75
|
+
resource_node_id=None,
|
|
76
|
+
executor="sql_endpoint_refresh",
|
|
77
|
+
payload=None,
|
|
78
|
+
payload_sha256=None,
|
|
79
|
+
),
|
|
80
|
+
),
|
|
81
|
+
),
|
|
82
|
+
),
|
|
83
|
+
)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Executor dispatch for build actions.
|
|
2
|
+
|
|
3
|
+
``spark_sql`` runs one create or frozen ``DROP``; ``spark_sql_batch`` runs an
|
|
4
|
+
ordered catalogue payload as one reported action. ``spark_schema`` makes one
|
|
5
|
+
schema, ``spark_table`` completes a table whose shape only the session knows,
|
|
6
|
+
and ``folder`` makes or removes a directory. ``tsql`` is the Warehouse SQL path.
|
|
7
|
+
``alias`` points one Lakehouse name at another item's object, and ``sql_endpoint``
|
|
8
|
+
syncs a Lakehouse's SQL analytics endpoint.
|
|
9
|
+
|
|
10
|
+
``load_file`` closes an item: it writes one file of the deployed runtime tree, or
|
|
11
|
+
removes one the source has stopped claiming. Generated load procedures need no
|
|
12
|
+
executor of their own — a create-or-alter is T-SQL, and ``tsql`` runs the payload
|
|
13
|
+
it is given without inspecting what the payload builds.
|
|
14
|
+
|
|
15
|
+
There is no prune executor — a build freezes its drops as payloads, so the
|
|
16
|
+
installer never enumerates the target.
|
|
17
|
+
|
|
18
|
+
Every Spark executor addresses the destination its batch names, and none of them
|
|
19
|
+
relies on what the session is attached to.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from .alias import AliasExecutor
|
|
25
|
+
from .base import ActionExecutor, InstallationContext, ResolvedTarget, SkippedExecution
|
|
26
|
+
from .folder import FolderExecutor
|
|
27
|
+
from .load_file import LoadFileExecutor
|
|
28
|
+
from .spark_schema import SparkSchemaExecutor
|
|
29
|
+
from .spark_sql import SparkSqlExecutor
|
|
30
|
+
from .spark_sql_batch import SparkSqlBatchExecutor
|
|
31
|
+
from .spark_table import SparkTableExecutor
|
|
32
|
+
from .sql_endpoint_refresh import SqlEndpointRefreshExecutor
|
|
33
|
+
from .tsql import TSqlBatchExecutor, TSqlExecutor
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def default_executors() -> dict[str, ActionExecutor]:
|
|
37
|
+
"""The executor registry, by name — the names actions carry."""
|
|
38
|
+
|
|
39
|
+
return {
|
|
40
|
+
SparkSqlExecutor.name: SparkSqlExecutor(),
|
|
41
|
+
SparkSqlBatchExecutor.name: SparkSqlBatchExecutor(),
|
|
42
|
+
SparkSchemaExecutor.name: SparkSchemaExecutor(),
|
|
43
|
+
SparkTableExecutor.name: SparkTableExecutor(),
|
|
44
|
+
FolderExecutor.name: FolderExecutor(),
|
|
45
|
+
LoadFileExecutor.name: LoadFileExecutor(),
|
|
46
|
+
TSqlExecutor.name: TSqlExecutor(),
|
|
47
|
+
TSqlBatchExecutor.name: TSqlBatchExecutor(),
|
|
48
|
+
AliasExecutor.name: AliasExecutor(),
|
|
49
|
+
SqlEndpointRefreshExecutor.name: SqlEndpointRefreshExecutor(),
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"ActionExecutor",
|
|
55
|
+
"InstallationContext",
|
|
56
|
+
"ResolvedTarget",
|
|
57
|
+
"AliasExecutor",
|
|
58
|
+
"SkippedExecution",
|
|
59
|
+
"SparkSchemaExecutor",
|
|
60
|
+
"SparkSqlExecutor",
|
|
61
|
+
"SparkSqlBatchExecutor",
|
|
62
|
+
"SparkTableExecutor",
|
|
63
|
+
"SqlEndpointRefreshExecutor",
|
|
64
|
+
"FolderExecutor",
|
|
65
|
+
"LoadFileExecutor",
|
|
66
|
+
"TSqlExecutor",
|
|
67
|
+
"TSqlBatchExecutor",
|
|
68
|
+
"default_executors",
|
|
69
|
+
]
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Materialising one Lakehouse alias — a pointer, in whatever form the host has.
|
|
2
|
+
|
|
3
|
+
The payload names two addresses and nothing else: this item's ``Schema.Object``,
|
|
4
|
+
and the object in another item it stands for. Both are resolved through the
|
|
5
|
+
environment, the same way every other action's destination is, so the bundle
|
|
6
|
+
carries no path from the machine that wrote it.
|
|
7
|
+
|
|
8
|
+
The *form* the pointer takes is transport:
|
|
9
|
+
|
|
10
|
+
``Fabric``
|
|
11
|
+
a OneLake shortcut in the destination Lakehouse, created through the
|
|
12
|
+
workspace's own API.
|
|
13
|
+
``the local emulator``
|
|
14
|
+
a filesystem link beside the destination's other tables, which is what makes
|
|
15
|
+
the emulator's ``Tables/`` area keep mirroring what a shortcut looks like in
|
|
16
|
+
OneLake — plus the catalogue registration Fabric performs for itself, because
|
|
17
|
+
a link no statement could name would not be an alias at all.
|
|
18
|
+
|
|
19
|
+
That is the same split :mod:`weaver.build_bundle.executors.spark_schema`
|
|
20
|
+
documents: which alias, over what, is settled and in the manifest; how a name is
|
|
21
|
+
made to point somewhere is the environment's business. An alias holds no data, so
|
|
22
|
+
an existing one is replaced rather than treated as an unexpected collision — a
|
|
23
|
+
build has to be able to run twice.
|
|
24
|
+
|
|
25
|
+
**The action is not finished until the alias can be read.** Fabric creates a
|
|
26
|
+
shortcut synchronously and *discovers* it asynchronously, so the API call
|
|
27
|
+
returning is not the same thing as the alias existing: for a few seconds the
|
|
28
|
+
Lakehouse reports the name as "neither a view nor a table". An action that
|
|
29
|
+
reported success there would make the barrier the plan puts around it a lie, and
|
|
30
|
+
the failure would surface in the next item's DDL — which is exactly where it did
|
|
31
|
+
surface, before this waited.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import json
|
|
37
|
+
import time
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
from ...errors import InstallError
|
|
41
|
+
from ...targets import DeltaTarget, FolderTarget
|
|
42
|
+
from ..models import BuildAction
|
|
43
|
+
from .base import InstallationContext, ResolvedTarget
|
|
44
|
+
from .spark_case import exact_identifier_case
|
|
45
|
+
|
|
46
|
+
FILES_AREA = "Files"
|
|
47
|
+
|
|
48
|
+
#: How long a freshly created shortcut may take to become addressable, and how
|
|
49
|
+
#: often to ask. Discovery normally takes seconds; the bound exists so a
|
|
50
|
+
#: never-appearing alias fails naming itself rather than as an obscure error in
|
|
51
|
+
#: whatever statement reads it next.
|
|
52
|
+
ADDRESSABLE_TIMEOUT = 300.0
|
|
53
|
+
ADDRESSABLE_POLL_INTERVAL = 5.0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class AliasExecutor:
|
|
57
|
+
name = "alias"
|
|
58
|
+
|
|
59
|
+
def execute(
|
|
60
|
+
self,
|
|
61
|
+
action: BuildAction,
|
|
62
|
+
payload: bytes | None,
|
|
63
|
+
context: InstallationContext,
|
|
64
|
+
) -> dict[str, Any] | None:
|
|
65
|
+
if payload is None:
|
|
66
|
+
raise InstallError(f"alias action {action.id!r} has no payload")
|
|
67
|
+
frozen = json.loads(payload.decode("utf-8"))["aliases"]
|
|
68
|
+
if not frozen:
|
|
69
|
+
return {"aliases": []}
|
|
70
|
+
|
|
71
|
+
shortcut = getattr(context.resolver, "create_onelake_shortcut", None)
|
|
72
|
+
link = getattr(context.store, "link", None)
|
|
73
|
+
if shortcut is None and link is None:
|
|
74
|
+
raise InstallError(
|
|
75
|
+
f"alias action {action.id!r} cannot be materialised here: this "
|
|
76
|
+
"environment offers neither a OneLake shortcut nor a store link"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
made = []
|
|
80
|
+
for each in frozen:
|
|
81
|
+
source = context.resolved(each["source_target_id"])
|
|
82
|
+
if shortcut is not None:
|
|
83
|
+
made.append(self._shortcut(shortcut, each, context, source))
|
|
84
|
+
else:
|
|
85
|
+
made.append(self._link(link, each, context, source))
|
|
86
|
+
|
|
87
|
+
details: dict[str, Any] = {"aliases": made}
|
|
88
|
+
# Every shortcut is created before anything waits, so the cost is one
|
|
89
|
+
# discovery window rather than one per alias.
|
|
90
|
+
if shortcut is not None and context.spark is not None:
|
|
91
|
+
waited = self._await_addressable(context, frozen)
|
|
92
|
+
if waited is not None:
|
|
93
|
+
details["addressable_after_seconds"] = waited
|
|
94
|
+
return details
|
|
95
|
+
|
|
96
|
+
def _shortcut(self, shortcut, frozen: dict, context, source) -> dict:
|
|
97
|
+
made = shortcut(
|
|
98
|
+
context.target.lakehouse,
|
|
99
|
+
path=f"{frozen['area']}/{frozen['schema']}",
|
|
100
|
+
name=frozen["object"],
|
|
101
|
+
source=source.lakehouse,
|
|
102
|
+
source_path=(
|
|
103
|
+
f"{frozen['source_area']}/{frozen['source_schema']}"
|
|
104
|
+
f"/{frozen['source_object']}"
|
|
105
|
+
),
|
|
106
|
+
)
|
|
107
|
+
return {"alias": frozen["alias"], "source": frozen["source"], **(made or {})}
|
|
108
|
+
|
|
109
|
+
def _link(self, link, frozen: dict, context, source) -> dict:
|
|
110
|
+
destination = _location(context.target, frozen, context, source=False)
|
|
111
|
+
producer = _location(source, frozen, context, source=True)
|
|
112
|
+
if not context.store.exists(producer):
|
|
113
|
+
raise InstallError(
|
|
114
|
+
f"alias {frozen['alias']} has no source to point at: "
|
|
115
|
+
f"{producer.value} does not exist"
|
|
116
|
+
)
|
|
117
|
+
if context.store.exists(destination):
|
|
118
|
+
context.store.delete(destination, recursive=True)
|
|
119
|
+
link(producer, destination)
|
|
120
|
+
made = {
|
|
121
|
+
"alias": frozen["alias"],
|
|
122
|
+
"source": frozen["source"],
|
|
123
|
+
"linked": destination.value,
|
|
124
|
+
"to": producer.value,
|
|
125
|
+
}
|
|
126
|
+
if frozen["area"] != FILES_AREA and context.spark is not None:
|
|
127
|
+
# Fabric discovers a shortcut under Tables/ and the table appears in
|
|
128
|
+
# the catalogue. Local Spark discovers nothing, so the emulator names
|
|
129
|
+
# it — otherwise the link would exist and no statement could reach it.
|
|
130
|
+
catalogue = context.catalogue
|
|
131
|
+
with exact_identifier_case(
|
|
132
|
+
context.spark,
|
|
133
|
+
enabled=catalogue.destination.preserve_table_identifier_case,
|
|
134
|
+
):
|
|
135
|
+
made["registered"] = catalogue.register_external_table(
|
|
136
|
+
frozen["schema"], frozen["object"], destination.value
|
|
137
|
+
)
|
|
138
|
+
return made
|
|
139
|
+
|
|
140
|
+
def _await_addressable(self, context: InstallationContext, frozen: list) -> float | None:
|
|
141
|
+
"""Wait until every table alias just created can actually be read.
|
|
142
|
+
|
|
143
|
+
The probe is a read, not a catalogue lookup, because the catalogue is the
|
|
144
|
+
thing that is briefly wrong: Fabric reports the shortcut's metadata
|
|
145
|
+
location while still refusing it as a relation. Only a read that succeeds
|
|
146
|
+
proves the alias is usable, and a read is what the next action does.
|
|
147
|
+
|
|
148
|
+
All of them are waited on together, so several aliases cost one discovery
|
|
149
|
+
window instead of one each.
|
|
150
|
+
"""
|
|
151
|
+
|
|
152
|
+
catalogue = context.catalogue
|
|
153
|
+
pending = {
|
|
154
|
+
each["alias"]: catalogue.qualify(each["schema"], each["object"])
|
|
155
|
+
for each in frozen
|
|
156
|
+
if each["area"] != FILES_AREA
|
|
157
|
+
}
|
|
158
|
+
if not pending:
|
|
159
|
+
return None
|
|
160
|
+
|
|
161
|
+
started = time.monotonic()
|
|
162
|
+
deadline = started + ADDRESSABLE_TIMEOUT
|
|
163
|
+
failure: Exception | None = None
|
|
164
|
+
while pending:
|
|
165
|
+
for alias, qualified in list(pending.items()):
|
|
166
|
+
try:
|
|
167
|
+
with exact_identifier_case(
|
|
168
|
+
context.spark,
|
|
169
|
+
enabled=catalogue.destination.preserve_table_identifier_case,
|
|
170
|
+
):
|
|
171
|
+
context.spark.sql(f"SELECT * FROM {qualified} LIMIT 0").collect()
|
|
172
|
+
del pending[alias]
|
|
173
|
+
except Exception as exc: # not discovered yet — or never will be
|
|
174
|
+
failure = exc
|
|
175
|
+
if not pending:
|
|
176
|
+
break
|
|
177
|
+
if time.monotonic() >= deadline:
|
|
178
|
+
raise InstallError(
|
|
179
|
+
f"alias(es) {', '.join(sorted(pending))} were created but did "
|
|
180
|
+
f"not become readable within {int(ADDRESSABLE_TIMEOUT)}s: {failure}"
|
|
181
|
+
) from failure
|
|
182
|
+
time.sleep(ADDRESSABLE_POLL_INTERVAL)
|
|
183
|
+
return round(time.monotonic() - started, 1)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _location(
|
|
187
|
+
target: ResolvedTarget,
|
|
188
|
+
frozen: dict,
|
|
189
|
+
context: InstallationContext,
|
|
190
|
+
*,
|
|
191
|
+
source: bool,
|
|
192
|
+
):
|
|
193
|
+
area = frozen["source_area"] if source else frozen["area"]
|
|
194
|
+
schema = frozen["source_schema"] if source else frozen["schema"]
|
|
195
|
+
name = frozen["source_object"] if source else frozen["object"]
|
|
196
|
+
if area == FILES_AREA:
|
|
197
|
+
return context.resolver.folder_object(
|
|
198
|
+
FolderTarget(lakehouse=target.lakehouse), schema, name
|
|
199
|
+
)
|
|
200
|
+
return context.resolver.delta_table(
|
|
201
|
+
DeltaTarget(lakehouse=target.lakehouse), schema, name
|
|
202
|
+
)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""The executor seam — dispatch only, no planning.
|
|
2
|
+
|
|
3
|
+
An executor runs one action's payload against one resolved target and returns
|
|
4
|
+
optional structured details, or raises. It never reads the repository, resolves
|
|
5
|
+
a dependency or selects a target: those decisions are all in the bundle already.
|
|
6
|
+
The installer owns timing, status and reporting; an executor owns the work.
|
|
7
|
+
|
|
8
|
+
The context carries runtime services — a Spark session, the resolver and store —
|
|
9
|
+
plus the one target the current batch is bound to. It carries no planning input,
|
|
10
|
+
and no way back to the repository: everything an action needs is its payload.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Any, Mapping, Protocol
|
|
17
|
+
|
|
18
|
+
from ...errors import InstallError
|
|
19
|
+
from ...locations import LakehouseSparkLocation
|
|
20
|
+
from ...spark import SparkCatalogue, SparkDestination
|
|
21
|
+
from ...store import Store
|
|
22
|
+
from ...targets import ItemRef
|
|
23
|
+
from ..models import BuildAction
|
|
24
|
+
from ..targets import BoundTarget
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class ResolvedTarget:
|
|
29
|
+
"""A manifest target resolved to what the executor addresses.
|
|
30
|
+
|
|
31
|
+
``lakehouse`` is the logical item the bundle named. The other two are that
|
|
32
|
+
item resolved into the two things Spark needs to reach it, and they are two
|
|
33
|
+
because Fabric answers them separately:
|
|
34
|
+
|
|
35
|
+
``location``
|
|
36
|
+
the physical roots — where the bytes are. An ``abfss://`` URL on Fabric, a
|
|
37
|
+
directory locally.
|
|
38
|
+
``destination``
|
|
39
|
+
the catalogue name — what a statement calls it. Fabric's four-part
|
|
40
|
+
``workspace.lakehouse.schema.object``; locally the folded database name.
|
|
41
|
+
|
|
42
|
+
Both are needed, and neither substitutes for the other: a folder is created at
|
|
43
|
+
a path and has no catalogue name, while a view exists only as a name and has no
|
|
44
|
+
path of its own.
|
|
45
|
+
|
|
46
|
+
Resolution happens here, once per target, rather than in each executor. An
|
|
47
|
+
executor that derived either for itself would be re-deciding where an action
|
|
48
|
+
lands, which is a planning decision it is not allowed to make. It is also what
|
|
49
|
+
lets one session build several destinations, and write the catalogue to a
|
|
50
|
+
different one again, without ever switching what the session is attached to.
|
|
51
|
+
|
|
52
|
+
Both are None for a Warehouse target, which is reached over TDS and has
|
|
53
|
+
neither.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
bound: BoundTarget
|
|
57
|
+
lakehouse: ItemRef
|
|
58
|
+
location: LakehouseSparkLocation | None = None
|
|
59
|
+
destination: SparkDestination | None = None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class InstallationContext:
|
|
64
|
+
"""Runtime services and the one target the current batch is bound to.
|
|
65
|
+
|
|
66
|
+
``spark`` runs Lakehouse work; ``sql`` runs Warehouse (T-SQL) work. A batch
|
|
67
|
+
names one target, so only the capability its actions need has to be present.
|
|
68
|
+
|
|
69
|
+
``targets`` holds every target the plan declared, already resolved. It exists
|
|
70
|
+
for the one action that legitimately spans two of them — an alias, which
|
|
71
|
+
points a name in ``target`` at an object in another — and it carries resolved
|
|
72
|
+
targets rather than ids so that a second destination is addressed exactly as
|
|
73
|
+
the batch's own is, and never derived by an executor.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
spark: Any
|
|
77
|
+
resolver: Any
|
|
78
|
+
store: Store
|
|
79
|
+
target: ResolvedTarget
|
|
80
|
+
sql: Any = None
|
|
81
|
+
targets: Mapping[str, ResolvedTarget] = field(default_factory=dict)
|
|
82
|
+
#: This installation's publication instant, resolved into ``{{epoch}}``. One
|
|
83
|
+
#: value for the whole run, so every Registry row a build writes carries the
|
|
84
|
+
#: same one and two rows can be ordered against each other. It is not a
|
|
85
|
+
#: destination's business, which is why it lives here rather than being
|
|
86
|
+
#: resolved with the object tokens.
|
|
87
|
+
epoch: str | None = None
|
|
88
|
+
|
|
89
|
+
def resolved(self, target_id: str) -> ResolvedTarget:
|
|
90
|
+
"""Another target this plan declared, by the id an action names."""
|
|
91
|
+
|
|
92
|
+
found = self.targets.get(target_id)
|
|
93
|
+
if found is None:
|
|
94
|
+
raise InstallError(
|
|
95
|
+
f"action names target {target_id!r}, which this plan does not declare"
|
|
96
|
+
)
|
|
97
|
+
return found
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def catalogue(self) -> SparkCatalogue:
|
|
101
|
+
"""Catalogue operations against *this batch's* destination.
|
|
102
|
+
|
|
103
|
+
Built per access rather than stored, so the context stays a frozen record
|
|
104
|
+
of what was resolved. Failing here — rather than falling back to the
|
|
105
|
+
session's own catalogue — is the point: an action with nowhere to go must
|
|
106
|
+
stop, not land somewhere plausible (how-does-build-work §4).
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
if self.target.destination is None:
|
|
110
|
+
raise InstallError(
|
|
111
|
+
f"target {self.target.bound.id!r} resolved to no Spark destination, "
|
|
112
|
+
"so a statement naming an object has nowhere to run"
|
|
113
|
+
)
|
|
114
|
+
return SparkCatalogue(self.spark, self.target.destination)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(frozen=True)
|
|
118
|
+
class SkippedExecution:
|
|
119
|
+
"""An executor's explicit, non-failing decision not to run on this host."""
|
|
120
|
+
|
|
121
|
+
details: dict[str, Any] | None = None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class ActionExecutor(Protocol):
|
|
125
|
+
name: str
|
|
126
|
+
|
|
127
|
+
def execute(
|
|
128
|
+
self,
|
|
129
|
+
action: BuildAction,
|
|
130
|
+
payload: bytes | None,
|
|
131
|
+
context: InstallationContext,
|
|
132
|
+
) -> dict[str, Any] | SkippedExecution | None: ...
|