weaverstack 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weaver/__init__.py +59 -0
- weaver/build_bundle/__init__.py +109 -0
- weaver/build_bundle/aliases.py +325 -0
- weaver/build_bundle/bundle.py +359 -0
- weaver/build_bundle/catalogue_actions.py +275 -0
- weaver/build_bundle/changes.py +186 -0
- weaver/build_bundle/endpoints.py +83 -0
- weaver/build_bundle/executors/__init__.py +69 -0
- weaver/build_bundle/executors/alias.py +202 -0
- weaver/build_bundle/executors/base.py +132 -0
- weaver/build_bundle/executors/folder.py +71 -0
- weaver/build_bundle/executors/load_file.py +205 -0
- weaver/build_bundle/executors/spark_case.py +26 -0
- weaver/build_bundle/executors/spark_schema.py +60 -0
- weaver/build_bundle/executors/spark_sql.py +59 -0
- weaver/build_bundle/executors/spark_sql_batch.py +57 -0
- weaver/build_bundle/executors/spark_table.py +213 -0
- weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
- weaver/build_bundle/executors/tsql.py +81 -0
- weaver/build_bundle/incremental.py +288 -0
- weaver/build_bundle/installer.py +384 -0
- weaver/build_bundle/models.py +288 -0
- weaver/build_bundle/payloads.py +34 -0
- weaver/build_bundle/physical.py +625 -0
- weaver/build_bundle/planner.py +389 -0
- weaver/build_bundle/prune.py +620 -0
- weaver/build_bundle/report.py +108 -0
- weaver/build_bundle/stages.py +196 -0
- weaver/build_bundle/targets.py +272 -0
- weaver/build_bundle/workflow.py +585 -0
- weaver/catalogue/__init__.py +73 -0
- weaver/catalogue/builtin.py +238 -0
- weaver/catalogue/claims.py +121 -0
- weaver/catalogue/projection.py +437 -0
- weaver/catalogue/reader.py +152 -0
- weaver/catalogue/reconcile.py +231 -0
- weaver/catalogue/render.py +410 -0
- weaver/catalogue/state.py +660 -0
- weaver/catalogue/tables.py +648 -0
- weaver/config.py +178 -0
- weaver/declaration/__init__.py +171 -0
- weaver/declaration/columns.py +223 -0
- weaver/declaration/ddl.py +266 -0
- weaver/declaration/dependencies.py +544 -0
- weaver/declaration/graph.py +240 -0
- weaver/declaration/item_dependencies.py +292 -0
- weaver/declaration/load.py +191 -0
- weaver/declaration/metadata.py +1405 -0
- weaver/declaration/model.py +448 -0
- weaver/declaration/references.py +294 -0
- weaver/declaration/repository.py +959 -0
- weaver/declaration/schemas.py +135 -0
- weaver/declaration/source.py +674 -0
- weaver/declaration/spark_load.py +759 -0
- weaver/declaration/sql_shaping.py +591 -0
- weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
- weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
- weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
- weaver/declaration/templates/load/column_metadata.sql +40 -0
- weaver/declaration/templates/load/full_replace_body.sql +21 -0
- weaver/declaration/templates/load/install_load_procedure.sql +27 -0
- weaver/declaration/templates/load/load_procedure.sql +48 -0
- weaver/declaration/templates/load/primary_key_body.sql +113 -0
- weaver/declaration/tsql_ddl.py +468 -0
- weaver/declaration/tsql_load.py +417 -0
- weaver/declaration/warehouse_type_mapping.yml +93 -0
- weaver/diagnostics.py +247 -0
- weaver/errors.py +61 -0
- weaver/etl.py +469 -0
- weaver/fabric/__init__.py +107 -0
- weaver/fabric/auth.py +137 -0
- weaver/fabric/capacity.py +143 -0
- weaver/fabric/client.py +147 -0
- weaver/fabric/environment.py +460 -0
- weaver/fabric/livy.py +478 -0
- weaver/fabric/notebooks.py +201 -0
- weaver/fabric/onelake.py +263 -0
- weaver/fabric/resolution.py +344 -0
- weaver/fabric/resources.py +245 -0
- weaver/fabric/session.py +148 -0
- weaver/fabric/shortcuts.py +120 -0
- weaver/fabric/sql.py +118 -0
- weaver/fabric/store.py +198 -0
- weaver/initialise.py +209 -0
- weaver/lakehouse.py +386 -0
- weaver/load.py +474 -0
- weaver/load_execution.py +483 -0
- weaver/load_plan.py +912 -0
- weaver/load_report.py +330 -0
- weaver/load_resolution.py +386 -0
- weaver/locations.py +164 -0
- weaver/objects.py +392 -0
- weaver/operations.py +757 -0
- weaver/physical_wipe.py +369 -0
- weaver/push.py +76 -0
- weaver/resolution.py +292 -0
- weaver/runtime/__init__.py +30 -0
- weaver/runtime/folder_load.py +402 -0
- weaver/runtime/load_contract.py +245 -0
- weaver/runtime/load_result.py +104 -0
- weaver/runtime/spark_load.py +152 -0
- weaver/runtime/table_load.py +497 -0
- weaver/spark/__init__.py +49 -0
- weaver/spark/catalogue.py +245 -0
- weaver/spark/destination.py +195 -0
- weaver/spark/session.py +84 -0
- weaver/spark/tokens.py +138 -0
- weaver/sql/__init__.py +40 -0
- weaver/sql/authentication.py +38 -0
- weaver/sql/connection.py +90 -0
- weaver/sql/errors.py +25 -0
- weaver/sql/execution.py +123 -0
- weaver/sql/pool.py +174 -0
- weaver/sql/wipe.py +156 -0
- weaver/store.py +209 -0
- weaver/targets.py +257 -0
- weaver/task_logging.py +215 -0
- weaver/unbind.py +74 -0
- weaver/workspaces.py +175 -0
- weaver_cli/__init__.py +12 -0
- weaver_cli/__main__.py +7 -0
- weaver_cli/main.py +626 -0
- weaverstack-0.1.1.dist-info/METADATA +113 -0
- weaverstack-0.1.1.dist-info/RECORD +127 -0
- weaverstack-0.1.1.dist-info/WHEEL +4 -0
- weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
- weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Spark catalogue operations, against a *named* destination.
|
|
2
|
+
|
|
3
|
+
Everything Weaver does to a Lakehouse through Spark goes through here, and the
|
|
4
|
+
one thing this type refuses to do is assume the session is pointed at the right
|
|
5
|
+
place. The session is attached to the Weaver Lakehouse — the fixed control plane
|
|
6
|
+
— so a destination is never the current catalogue, and an operation that did not
|
|
7
|
+
name one would land in the control plane instead. That is the failure this
|
|
8
|
+
exists to make impossible, not merely unlikely.
|
|
9
|
+
|
|
10
|
+
A build has at least two of these open at once: one for the Lakehouse being
|
|
11
|
+
built, one for the Weaver Lakehouse the catalogue is written to. They differ only
|
|
12
|
+
in their :class:`~weaver.spark.destination.SparkDestination`, which is the claim
|
|
13
|
+
being made — that once a name is right, the operation is the same one.
|
|
14
|
+
|
|
15
|
+
**Enumerating a destination's schemas is not here, and that is deliberate.** On
|
|
16
|
+
Fabric it cannot be: a schema is a three-level name under ``spark_catalog``, and
|
|
17
|
+
``SHOW SCHEMAS IN `workspace`.`lakehouse``` is refused — a bare ``SHOW SCHEMAS``
|
|
18
|
+
lists the *attached* Lakehouse and nothing else. Schema discovery therefore reads
|
|
19
|
+
the destination's ``Tables/`` area through the store, which works across
|
|
20
|
+
Lakehouses on both workspaces and is what prune already does. Offering a
|
|
21
|
+
``list_schemas`` here that silently answered for the wrong Lakehouse would be the
|
|
22
|
+
ambient-context mistake wearing an abstraction (how-does-build-work §4).
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from ..errors import InstallError
|
|
30
|
+
from . import tokens
|
|
31
|
+
from .destination import SparkDestination
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class SparkCatalogue:
|
|
35
|
+
"""Catalogue operations against one logical Spark destination."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, spark: Any, destination: SparkDestination) -> None:
|
|
38
|
+
if spark is None:
|
|
39
|
+
raise InstallError(
|
|
40
|
+
f"a Spark session is needed to reach {destination.item!r}, "
|
|
41
|
+
"and none was provided"
|
|
42
|
+
)
|
|
43
|
+
self.spark = spark
|
|
44
|
+
self.destination = destination
|
|
45
|
+
if destination.case_sensitive_analysis:
|
|
46
|
+
# Local's folded schema is lower-case and every declared object keeps
|
|
47
|
+
# its exact Weaver spelling. Unlike Fabric's catalogue, Spark's local
|
|
48
|
+
# session catalogue cannot look that object up again after reverting
|
|
49
|
+
# to case-insensitive analysis, so this is the emulator's session
|
|
50
|
+
# policy rather than a one-statement override.
|
|
51
|
+
spark.conf.set("spark.sql.caseSensitive", "true")
|
|
52
|
+
|
|
53
|
+
# --- naming -----------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
def qualify(self, schema: str, name: str) -> str:
|
|
56
|
+
return self.destination.qualify(schema, name)
|
|
57
|
+
|
|
58
|
+
def qualified_schema(self, schema: str) -> str:
|
|
59
|
+
return self.destination.qualified_schema(schema)
|
|
60
|
+
|
|
61
|
+
def expand(self, statement: str) -> str:
|
|
62
|
+
"""One payload's object tokens, resolved to this destination."""
|
|
63
|
+
|
|
64
|
+
return tokens.expand(statement, self.destination)
|
|
65
|
+
|
|
66
|
+
# --- execution ---------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
def sql(self, statement: str) -> Any:
|
|
69
|
+
"""Run one statement here, with its object tokens resolved first."""
|
|
70
|
+
|
|
71
|
+
return self.spark.sql(self.expand(statement))
|
|
72
|
+
|
|
73
|
+
# --- structure ---------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
def create_schema(self, schema: str, *, if_not_exists: bool = True) -> str:
|
|
76
|
+
"""Create a schema in this destination, and return the statement run.
|
|
77
|
+
|
|
78
|
+
The ``LOCATION`` clause is the destination's business, not the planner's:
|
|
79
|
+
local Spark needs one so a managed table lands under the Lakehouse's
|
|
80
|
+
``Tables`` area, and a schema-enabled Fabric Lakehouse pins it natively and
|
|
81
|
+
must not be given one. It is also a resolved path, so it could not have
|
|
82
|
+
been frozen into a payload without tying the bundle to the machine that
|
|
83
|
+
generated it (how-does-build-work §15).
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
qualifier = " IF NOT EXISTS" if if_not_exists else ""
|
|
87
|
+
statement = f"CREATE SCHEMA{qualifier} {self.qualified_schema(schema)}"
|
|
88
|
+
location = self.destination.schema_location(schema)
|
|
89
|
+
if location is not None:
|
|
90
|
+
statement += f" LOCATION '{_escaped(location)}'"
|
|
91
|
+
self.spark.sql(statement)
|
|
92
|
+
return statement
|
|
93
|
+
|
|
94
|
+
def register_external_table(self, schema: str, name: str, location: str) -> str:
|
|
95
|
+
"""Name a table in this destination whose storage it does not own.
|
|
96
|
+
|
|
97
|
+
This exists for one thing: an alias in the local emulator. Fabric
|
|
98
|
+
discovers a OneLake shortcut placed under a Lakehouse's ``Tables`` area by
|
|
99
|
+
itself, and the table simply appears in the catalogue; local Spark
|
|
100
|
+
discovers nothing, so the emulator has to say out loud what Fabric infers.
|
|
101
|
+
|
|
102
|
+
Unregistered first rather than created strictly, because an alias is a
|
|
103
|
+
pointer and re-pointing one is not a destructive transition. Dropping an
|
|
104
|
+
*external* table removes the registration and never the storage, which is
|
|
105
|
+
exactly the distinction that makes this safe: the data belongs to the item
|
|
106
|
+
that produced it.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
qualified = self.qualify(schema, name)
|
|
110
|
+
self.spark.sql(f"DROP TABLE IF EXISTS {qualified}")
|
|
111
|
+
statement = (
|
|
112
|
+
f"CREATE TABLE {qualified} USING DELTA LOCATION '{_escaped(location)}'"
|
|
113
|
+
)
|
|
114
|
+
self.spark.sql(statement)
|
|
115
|
+
return statement
|
|
116
|
+
|
|
117
|
+
# --- discovery ---------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
def schema_exists(self, schema: str) -> bool:
|
|
120
|
+
return bool(self.spark.catalog.databaseExists(self.qualified_schema(schema)))
|
|
121
|
+
|
|
122
|
+
def views(self, schema: str) -> tuple[str, ...]:
|
|
123
|
+
"""Persistent view names in one schema of this destination.
|
|
124
|
+
|
|
125
|
+
Views are catalogue-only — there is no directory to find them in — so this
|
|
126
|
+
is the one part of an inventory that has to be asked of Spark.
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
rows = self._rows(f"SHOW VIEWS IN {self.qualified_schema(schema)}")
|
|
130
|
+
names = []
|
|
131
|
+
for row in rows:
|
|
132
|
+
data = row.asDict()
|
|
133
|
+
if data.get("isTemporary"):
|
|
134
|
+
continue
|
|
135
|
+
name = data.get("viewName") or data.get("name")
|
|
136
|
+
if name:
|
|
137
|
+
names.append(name)
|
|
138
|
+
return tuple(names)
|
|
139
|
+
|
|
140
|
+
def tables(self, schema: str) -> tuple[str, ...]:
|
|
141
|
+
"""Table names in one schema of this destination.
|
|
142
|
+
|
|
143
|
+
``SHOW TABLES`` returns views as well, so the views are taken back out.
|
|
144
|
+
Prune does not use this — a Delta table is a directory, and reading the
|
|
145
|
+
storage is what keeps reconciliation scoped to the one Lakehouse — but a
|
|
146
|
+
test asserting what a build actually created needs to ask the catalogue,
|
|
147
|
+
not the filesystem.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
views = {name.lower() for name in self.views(schema)}
|
|
151
|
+
rows = self._rows(f"SHOW TABLES IN {self.qualified_schema(schema)}")
|
|
152
|
+
names = []
|
|
153
|
+
for row in rows:
|
|
154
|
+
data = row.asDict()
|
|
155
|
+
if data.get("isTemporary"):
|
|
156
|
+
continue
|
|
157
|
+
name = data.get("tableName") or data.get("name")
|
|
158
|
+
if name and name.lower() not in views:
|
|
159
|
+
names.append(name)
|
|
160
|
+
return tuple(names)
|
|
161
|
+
|
|
162
|
+
def exists(self, schema: str, name: str) -> bool:
|
|
163
|
+
"""Whether one object exists in this destination, table or view."""
|
|
164
|
+
|
|
165
|
+
return bool(self.spark.catalog.tableExists(self.qualify(schema, name)))
|
|
166
|
+
|
|
167
|
+
def _rows(self, statement: str) -> list:
|
|
168
|
+
"""Run a listing, reading an absent schema as an empty one.
|
|
169
|
+
|
|
170
|
+
A schema that is not there holds nothing, which is the answer an inventory
|
|
171
|
+
wants — and both workspaces raise for it rather than returning no rows. So the
|
|
172
|
+
absence is tolerated and everything else propagates, narrowly, for the same
|
|
173
|
+
reason :mod:`weaver.catalogue.reader` does it that way: a real failure read
|
|
174
|
+
as "nothing here" tells the next build that nothing is managed.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
try:
|
|
178
|
+
return self.spark.sql(statement).collect()
|
|
179
|
+
except Exception as exception:
|
|
180
|
+
if _is_missing_schema(exception):
|
|
181
|
+
return []
|
|
182
|
+
raise
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
#: Spark's error class for a namespace that does not exist. A missing Lakehouse
|
|
186
|
+
#: reports the same one, which is what we want: an inventory of somewhere that is
|
|
187
|
+
#: not there is empty either way.
|
|
188
|
+
_ABSENT = frozenset({"SCHEMA_NOT_FOUND"})
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _is_missing_schema(exception: Exception) -> bool:
|
|
192
|
+
"""Whether this means "not created yet" rather than "went wrong".
|
|
193
|
+
|
|
194
|
+
Keyed on Spark's error class, not on message text, so a reworded message
|
|
195
|
+
cannot quietly turn an infrastructure failure into an empty inventory. The
|
|
196
|
+
class name is consulted only when no error class is available — a stub session
|
|
197
|
+
in a test, or a connector that raises a plain error.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
error_class = getattr(exception, "getErrorClass", None)
|
|
201
|
+
if callable(error_class):
|
|
202
|
+
try:
|
|
203
|
+
found = error_class()
|
|
204
|
+
except Exception: # pragma: no cover - a broken accessor is not absence
|
|
205
|
+
found = None
|
|
206
|
+
if found:
|
|
207
|
+
return found in _ABSENT
|
|
208
|
+
return any(name in str(exception) for name in _ABSENT) or (
|
|
209
|
+
"NoSuchNamespaceException" in type(exception).__name__
|
|
210
|
+
or "NoSuchDatabaseException" in type(exception).__name__
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _escaped(value: str) -> str:
|
|
215
|
+
return value.replace("\\", "\\\\").replace("'", "\\'")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def drop_local_destination_catalogue(
|
|
219
|
+
spark: Any, destination: SparkDestination
|
|
220
|
+
) -> tuple[str, ...]:
|
|
221
|
+
"""Forget every namespace folded beneath one emulated Lakehouse.
|
|
222
|
+
|
|
223
|
+
Local CLI sessions use a persistent metastore so a later process can see
|
|
224
|
+
what ``initialise`` and ``build`` registered. A local wipe must therefore
|
|
225
|
+
clear catalogue registrations as well as the Fabric-shaped filesystem tree;
|
|
226
|
+
Fabric performs that bookkeeping itself when its Lakehouse storage is
|
|
227
|
+
emptied and never calls this emulator-only primitive.
|
|
228
|
+
"""
|
|
229
|
+
|
|
230
|
+
if destination.namespace or not destination.schema_prefix:
|
|
231
|
+
raise InstallError("local catalogue cleanup needs a folded local destination")
|
|
232
|
+
rows = spark.sql("SHOW DATABASES").collect()
|
|
233
|
+
prefix = destination.schema_prefix.casefold()
|
|
234
|
+
schemas = []
|
|
235
|
+
for row in rows:
|
|
236
|
+
data = row.asDict() if hasattr(row, "asDict") else {}
|
|
237
|
+
name = data.get("namespace") or data.get("databaseName") or data.get("schemaName")
|
|
238
|
+
if name and str(name).casefold().startswith(prefix):
|
|
239
|
+
schemas.append(str(name))
|
|
240
|
+
statements = []
|
|
241
|
+
for schema in sorted(schemas, key=str.casefold):
|
|
242
|
+
statement = f"DROP SCHEMA IF EXISTS {destination.qualified_schema(schema[len(destination.schema_prefix):])} CASCADE"
|
|
243
|
+
spark.sql(statement)
|
|
244
|
+
statements.append(statement)
|
|
245
|
+
return tuple(statements)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""How one logical Lakehouse is *named* in a Spark session.
|
|
2
|
+
|
|
3
|
+
:class:`~weaver.locations.LakehouseSparkLocation` answers where a destination's
|
|
4
|
+
bytes live. This answers the other half: what a statement has to write to reach
|
|
5
|
+
that destination's catalogue, given that the session is attached somewhere else.
|
|
6
|
+
|
|
7
|
+
The two workspaces disagree, and the disagreement is **data**, not behaviour — which
|
|
8
|
+
is why there is one class here and two constructors rather than two classes.
|
|
9
|
+
|
|
10
|
+
**Fabric** has a native namespace for exactly this. Under ``spark_catalog`` a
|
|
11
|
+
schema is a three-level name, so an object is four parts::
|
|
12
|
+
|
|
13
|
+
`Weaver`.`Play_Lakehouse_1`.`Sales`.`Customer`
|
|
14
|
+
^workspace ^lakehouse ^schema ^object
|
|
15
|
+
|
|
16
|
+
One session can create, read and drop through that name in any Lakehouse in the
|
|
17
|
+
workspace, and can build a view in one over a table in another. Nothing has to be
|
|
18
|
+
attached, and nothing has to be switched.
|
|
19
|
+
|
|
20
|
+
**Local Spark** has no such namespace, and cannot be given one: a Delta catalogue
|
|
21
|
+
can only be the *session* catalogue (``DeltaCatalog`` extends
|
|
22
|
+
``DelegatingCatalogExtension``, and registered as an ordinary named catalogue its
|
|
23
|
+
delegate is null), so ``spark.sql.catalog.<lakehouse>`` is not available. Its one
|
|
24
|
+
namespace level is the database. The proxy therefore folds the Lakehouse into
|
|
25
|
+
that level::
|
|
26
|
+
|
|
27
|
+
`sales_lh__sales`.`Customer`
|
|
28
|
+
|
|
29
|
+
which is not Fabric syntax and is not meant to be. What it reproduces is the
|
|
30
|
+
*property* Fabric's namespace provides and a bare ``Sales.Customer`` does not:
|
|
31
|
+
two destinations that declare a schema of the same name stay apart. Storage is
|
|
32
|
+
untouched by the folding — the database still carries an explicit ``LOCATION`` of
|
|
33
|
+
``<lakehouse>/Tables/<schema>``, so a managed table lands exactly where the
|
|
34
|
+
Fabric layout puts it and the emulator keeps mirroring OneLake. The folded
|
|
35
|
+
database identifier is lower-case because the local session catalogue registers
|
|
36
|
+
it that way; declared object identifiers remain exact-case under the emulator's
|
|
37
|
+
case-sensitive analysis policy.
|
|
38
|
+
|
|
39
|
+
A destination is never carried in a build bundle. It is derived at install time
|
|
40
|
+
from the item the bundle names, because a Fabric namespace is workspace-specific
|
|
41
|
+
and a local one is rooted in a temporary directory — see
|
|
42
|
+
:class:`~weaver.locations.LakehouseSparkLocation` for why a bundle that moved
|
|
43
|
+
with either would stop being comparable between environments
|
|
44
|
+
(how-does-build-work §15).
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import re
|
|
50
|
+
from dataclasses import dataclass
|
|
51
|
+
|
|
52
|
+
from ..errors import IdentityError
|
|
53
|
+
|
|
54
|
+
#: Local Spark accepts word characters in a database name and nothing else, so a
|
|
55
|
+
#: folded name is only legal if the Lakehouse's own name is.
|
|
56
|
+
_LEGAL_LOCAL_NAME = re.compile(r"\A\w+\Z")
|
|
57
|
+
|
|
58
|
+
#: What separates the Lakehouse from the schema in the folded local name. Two
|
|
59
|
+
#: characters, so a single underscore in either half cannot be mistaken for it.
|
|
60
|
+
LOCAL_SEPARATOR = "__"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def identifier(name: str) -> str:
|
|
64
|
+
"""A back-tick quoted Spark identifier, safe for spaces and keywords."""
|
|
65
|
+
|
|
66
|
+
return "`" + name.replace("`", "``") + "`"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True)
|
|
70
|
+
class SparkDestination:
|
|
71
|
+
"""One logical Lakehouse, as a Spark session addresses it.
|
|
72
|
+
|
|
73
|
+
``item`` is the logical name, and is what appears in a message. ``namespace``
|
|
74
|
+
is whatever sits above the schema — the workspace and Lakehouse on Fabric,
|
|
75
|
+
nothing locally. ``schema_prefix`` is the local fold, empty on Fabric.
|
|
76
|
+
|
|
77
|
+
``tables_root`` is present only for the platform that needs it: local Spark
|
|
78
|
+
drops a managed table into its own warehouse directory unless the database
|
|
79
|
+
says otherwise, so the proxy supplies a ``LOCATION``. A schema-enabled Fabric
|
|
80
|
+
Lakehouse pins its own, and answers None.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
item: str
|
|
84
|
+
namespace: tuple[str, ...] = ()
|
|
85
|
+
schema_prefix: str = ""
|
|
86
|
+
tables_root: str | None = None
|
|
87
|
+
preserve_table_identifier_case: bool = False
|
|
88
|
+
lowercase_schema_identifier: bool = False
|
|
89
|
+
case_sensitive_analysis: bool = False
|
|
90
|
+
|
|
91
|
+
def schema_identifier(self, schema: str) -> str:
|
|
92
|
+
"""The schema's name at its own namespace level, unquoted."""
|
|
93
|
+
|
|
94
|
+
name = f"{self.schema_prefix}{_checked(schema, what='schema')}"
|
|
95
|
+
return name.lower() if self.lowercase_schema_identifier else name
|
|
96
|
+
|
|
97
|
+
def qualified_schema(self, schema: str) -> str:
|
|
98
|
+
"""The schema, fully qualified — what ``CREATE SCHEMA`` is given."""
|
|
99
|
+
|
|
100
|
+
return ".".join(
|
|
101
|
+
identifier(part)
|
|
102
|
+
for part in (*self.namespace, self.schema_identifier(schema))
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def qualify(self, schema: str, name: str) -> str:
|
|
106
|
+
"""One object, fully qualified — what every statement names it by."""
|
|
107
|
+
|
|
108
|
+
return (
|
|
109
|
+
f"{self.qualified_schema(schema)}"
|
|
110
|
+
f".{identifier(_checked(name, what='object name'))}"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def schema_location(self, schema: str) -> str | None:
|
|
114
|
+
"""Where this destination's managed tables for a schema must be pinned.
|
|
115
|
+
|
|
116
|
+
None when the platform pins them itself, which is the Fabric answer and
|
|
117
|
+
the reason no path reaches a Fabric ``CREATE SCHEMA``.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
if self.tables_root is None:
|
|
121
|
+
return None
|
|
122
|
+
return f"{self.tables_root.rstrip('/')}/{_checked(schema, what='schema')}"
|
|
123
|
+
|
|
124
|
+
def __str__(self) -> str:
|
|
125
|
+
return self.item if not self.namespace else ".".join(self.namespace)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def fabric_destination(*, workspace: str, lakehouse: str) -> SparkDestination:
|
|
129
|
+
"""A Fabric Lakehouse, addressed by its native four-part name.
|
|
130
|
+
|
|
131
|
+
Both names are display names, deliberately: this is what Fabric's Spark
|
|
132
|
+
namespace is spelled with, and it is what a reviewer reading a statement can
|
|
133
|
+
recognise. The workspace and item *ids* stay where they belong — in
|
|
134
|
+
resolution, and in the bundle's target block.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
return SparkDestination(
|
|
138
|
+
item=lakehouse,
|
|
139
|
+
namespace=(
|
|
140
|
+
_checked(workspace, what="workspace"),
|
|
141
|
+
_checked(lakehouse, what="lakehouse"),
|
|
142
|
+
),
|
|
143
|
+
# Fabric otherwise folds a quoted table identifier to lower-case at
|
|
144
|
+
# creation.
|
|
145
|
+
preserve_table_identifier_case=True,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def local_destination(*, item: str, tables_root: str) -> SparkDestination:
|
|
150
|
+
"""A local Lakehouse, folded into the one namespace level Spark offers.
|
|
151
|
+
|
|
152
|
+
The Lakehouse name becomes part of every database name, so it has to be a
|
|
153
|
+
legal one. Refused rather than sanitised: a silently altered name would make
|
|
154
|
+
two destinations collide again, which is the single thing this exists to
|
|
155
|
+
prevent.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
name = _checked(item, what="lakehouse")
|
|
159
|
+
if not _LEGAL_LOCAL_NAME.match(name):
|
|
160
|
+
raise IdentityError(
|
|
161
|
+
f"local Spark folds the Lakehouse name into its database names, and "
|
|
162
|
+
f"only accepts letters, digits and underscores there — {item!r} cannot "
|
|
163
|
+
"be addressed locally"
|
|
164
|
+
)
|
|
165
|
+
return SparkDestination(
|
|
166
|
+
item=name,
|
|
167
|
+
schema_prefix=f"{name}{LOCAL_SEPARATOR}",
|
|
168
|
+
tables_root=tables_root,
|
|
169
|
+
# The emulator mirrors Fabric's case-preserving table directories. Its
|
|
170
|
+
# folded schema itself was registered under Spark's case-insensitive
|
|
171
|
+
# policy, so every statement addresses it by its canonical lower case.
|
|
172
|
+
preserve_table_identifier_case=True,
|
|
173
|
+
lowercase_schema_identifier=True,
|
|
174
|
+
# Unlike Fabric's catalogue, Spark's local session catalogue cannot find
|
|
175
|
+
# a PascalCase table again after analysis returns to case-insensitive
|
|
176
|
+
# mode. The emulator therefore uses one exact-case policy for its life.
|
|
177
|
+
case_sensitive_analysis=True,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _checked(value: object, *, what: str) -> str:
|
|
182
|
+
"""One name part, checked rather than trusted.
|
|
183
|
+
|
|
184
|
+
These strings are concatenated into identifiers and into paths, so a part
|
|
185
|
+
carrying a delimiter would name something other than what it says.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
if not isinstance(value, str):
|
|
189
|
+
raise IdentityError(f"{what} must be a string, got {type(value).__name__}")
|
|
190
|
+
name = value.strip()
|
|
191
|
+
if not name:
|
|
192
|
+
raise IdentityError(f"{what} must not be empty")
|
|
193
|
+
if "." in name or "/" in name or "\\" in name:
|
|
194
|
+
raise IdentityError(f"{what} must not contain a separator: {value!r}")
|
|
195
|
+
return name
|
weaver/spark/session.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""A leak-free local Delta session for CLI execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from importlib import import_module
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Iterator
|
|
11
|
+
|
|
12
|
+
from ..diagnostics import SUPPORTED_JAVA, find_java_home
|
|
13
|
+
from ..errors import CommandError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@contextmanager
|
|
17
|
+
def local_delta_session(workspace=None) -> Iterator[object]:
|
|
18
|
+
"""Create one local Delta session and always stop it before returning.
|
|
19
|
+
|
|
20
|
+
CLI invocations are separate JVMs. When a local Workspace is supplied its
|
|
21
|
+
Spark metastore therefore lives beneath that emulator root, so namespaces
|
|
22
|
+
and table registrations created by ``initialise`` remain visible to the next
|
|
23
|
+
``build`` command just as Fabric's catalogue remains visible between
|
|
24
|
+
sessions. Tests that supply no Workspace keep Spark's process-local default.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
configure_spark_with_delta_pip = import_module(
|
|
29
|
+
"delta"
|
|
30
|
+
).configure_spark_with_delta_pip
|
|
31
|
+
SparkSession = import_module("pyspark.sql").SparkSession
|
|
32
|
+
except ImportError as exc:
|
|
33
|
+
raise CommandError(
|
|
34
|
+
"local build needs the Spark extra; install weaverstack[spark]"
|
|
35
|
+
) from exc
|
|
36
|
+
|
|
37
|
+
java_home = find_java_home()
|
|
38
|
+
if java_home is None:
|
|
39
|
+
raise CommandError(
|
|
40
|
+
f"local build needs Java {' or '.join(SUPPORTED_JAVA)}; run weaver doctor"
|
|
41
|
+
)
|
|
42
|
+
previous = {
|
|
43
|
+
name: os.environ.get(name)
|
|
44
|
+
for name in ("JAVA_HOME", "PYSPARK_PYTHON", "PYSPARK_DRIVER_PYTHON")
|
|
45
|
+
}
|
|
46
|
+
os.environ["JAVA_HOME"] = java_home
|
|
47
|
+
os.environ["PYSPARK_PYTHON"] = sys.executable
|
|
48
|
+
os.environ["PYSPARK_DRIVER_PYTHON"] = sys.executable
|
|
49
|
+
builder = (
|
|
50
|
+
SparkSession.builder.appName("weaverstack-cli")
|
|
51
|
+
.master("local[*]")
|
|
52
|
+
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
|
|
53
|
+
.config(
|
|
54
|
+
"spark.sql.catalog.spark_catalog",
|
|
55
|
+
"org.apache.spark.sql.delta.catalog.DeltaCatalog",
|
|
56
|
+
)
|
|
57
|
+
.config("spark.ui.enabled", "false")
|
|
58
|
+
.config("spark.sql.shuffle.partitions", "1")
|
|
59
|
+
.config("spark.databricks.delta.snapshotPartitions", "1")
|
|
60
|
+
)
|
|
61
|
+
if workspace is not None:
|
|
62
|
+
root_value = getattr(workspace, "workspace", workspace)
|
|
63
|
+
root = Path(root_value).expanduser().resolve() / ".weaver" / "spark"
|
|
64
|
+
builder = (
|
|
65
|
+
builder.config("spark.sql.catalogImplementation", "hive")
|
|
66
|
+
.config("spark.sql.warehouse.dir", str(root / "warehouse"))
|
|
67
|
+
.config(
|
|
68
|
+
"javax.jdo.option.ConnectionURL",
|
|
69
|
+
f"jdbc:derby:;databaseName={root / 'metastore'};create=true",
|
|
70
|
+
)
|
|
71
|
+
)
|
|
72
|
+
session = None
|
|
73
|
+
try:
|
|
74
|
+
session = configure_spark_with_delta_pip(builder).getOrCreate()
|
|
75
|
+
session.sparkContext.setLogLevel("ERROR")
|
|
76
|
+
yield session
|
|
77
|
+
finally:
|
|
78
|
+
if session is not None:
|
|
79
|
+
session.stop()
|
|
80
|
+
for name, value in previous.items():
|
|
81
|
+
if value is None:
|
|
82
|
+
os.environ.pop(name, None)
|
|
83
|
+
else:
|
|
84
|
+
os.environ[name] = value
|
weaver/spark/tokens.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""How a frozen payload names an object without naming a destination.
|
|
2
|
+
|
|
3
|
+
A generated statement has to say *which* object it acts on. It must not say
|
|
4
|
+
which Lakehouse, in which workspace, at which path: that is bound by the batch's
|
|
5
|
+
target, resolved at install time, and different in every environment. Writing it
|
|
6
|
+
into the SQL would make two bundles of the same repository differ in every
|
|
7
|
+
payload merely for having been generated somewhere else, which is exactly the
|
|
8
|
+
comparison how-does-build-work §15 exists to protect.
|
|
9
|
+
|
|
10
|
+
So a payload names an object logically, and the executor asks the batch's
|
|
11
|
+
destination what that is called there::
|
|
12
|
+
|
|
13
|
+
CREATE VIEW {{object:Sales.ActiveCustomer}} AS
|
|
14
|
+
SELECT * FROM {{object:Sales.Customer}} WHERE IsActive
|
|
15
|
+
|
|
16
|
+
Fabric -> `Weaver`.`Play_Lakehouse_1`.`Sales`.`ActiveCustomer`
|
|
17
|
+
local -> `sales_lh__sales`.`ActiveCustomer`
|
|
18
|
+
|
|
19
|
+
This is substitution of a transport-level value, not a template (§16). Nothing
|
|
20
|
+
semantic is left for the installer to decide: the object, its schema, the
|
|
21
|
+
statement and the destination are all fixed before the bundle is written — the
|
|
22
|
+
only thing supplied late is how that already-chosen destination spells a name.
|
|
23
|
+
A reviewer reading the payload sees the object; the manifest's target block says
|
|
24
|
+
where it goes. A bare two-part name said neither, and resolved through whatever
|
|
25
|
+
the session happened to be attached to.
|
|
26
|
+
|
|
27
|
+
The tokens are deliberately unmistakable. ``{{`` and ``}}`` are not Spark SQL, so
|
|
28
|
+
an unexpanded one is a syntax error at the point of use rather than a name that
|
|
29
|
+
quietly resolves somewhere else.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import re
|
|
35
|
+
|
|
36
|
+
from ..errors import InstallError
|
|
37
|
+
from .destination import SparkDestination
|
|
38
|
+
|
|
39
|
+
#: ``{{object:Schema.Name}}`` — one managed object.
|
|
40
|
+
OBJECT = re.compile(r"\{\{object:([^.{}]+)\.([^.{}]+)\}\}")
|
|
41
|
+
|
|
42
|
+
#: ``{{schema:Name}}`` — one managed schema.
|
|
43
|
+
SCHEMA = re.compile(r"\{\{schema:([^.{}]+)\}\}")
|
|
44
|
+
|
|
45
|
+
#: ``{{epoch}}`` — the instant this installation published its Registry.
|
|
46
|
+
#:
|
|
47
|
+
#: The one token that is not about a destination, which is why :func:`expand`
|
|
48
|
+
#: does not resolve it: it is scoped to the *installation*, and every statement
|
|
49
|
+
#: in a build must receive the same value however many destinations they name.
|
|
50
|
+
#: :func:`substitute_epoch` puts it in, and it has to run first — ``expand``
|
|
51
|
+
#: rejects any token it does not recognise, so an epoch that reached it would be
|
|
52
|
+
#: an error rather than silently surviving into the engine.
|
|
53
|
+
#:
|
|
54
|
+
#: It is a token rather than a literal frozen at generation time for the reason
|
|
55
|
+
#: this whole module exists: a rendered clock would make the same repository
|
|
56
|
+
#: produce different payload bytes on every run, and a bundle's identity is its
|
|
57
|
+
#: bytes.
|
|
58
|
+
EPOCH = re.compile(r"\{\{epoch\}\}")
|
|
59
|
+
|
|
60
|
+
#: The payload spelling of the publication epoch.
|
|
61
|
+
EPOCH_TOKEN = "{{epoch}}"
|
|
62
|
+
|
|
63
|
+
#: Anything else in token shape. Matched only so an unknown one is reported
|
|
64
|
+
#: rather than passed through to the engine as mystery syntax.
|
|
65
|
+
ANY = re.compile(r"\{\{[^{}]*\}\}")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def object_token(schema: str, name: str) -> str:
|
|
69
|
+
"""The payload spelling of one object."""
|
|
70
|
+
|
|
71
|
+
return f"{{{{object:{_part(schema, 'schema')}.{_part(name, 'object name')}}}}}"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def schema_token(schema: str) -> str:
|
|
75
|
+
"""The payload spelling of one schema."""
|
|
76
|
+
|
|
77
|
+
return f"{{{{schema:{_part(schema, 'schema')}}}}}"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def expand(text: str, destination: SparkDestination) -> str:
|
|
81
|
+
"""Every token in ``text``, resolved against one destination.
|
|
82
|
+
|
|
83
|
+
An unrecognised token is an error. Leaving it in place would hand Spark
|
|
84
|
+
something it cannot parse — better — or, if the shape ever became valid
|
|
85
|
+
syntax, something that means the wrong thing — far worse.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
text = OBJECT.sub(
|
|
89
|
+
lambda match: destination.qualify(match.group(1), match.group(2)), text
|
|
90
|
+
)
|
|
91
|
+
text = SCHEMA.sub(
|
|
92
|
+
lambda match: destination.qualified_schema(match.group(1)), text
|
|
93
|
+
)
|
|
94
|
+
leftover = ANY.search(text)
|
|
95
|
+
if leftover:
|
|
96
|
+
raise InstallError(
|
|
97
|
+
f"{leftover.group(0)} is not a name this installer knows how to "
|
|
98
|
+
f"resolve against {destination.item!r}"
|
|
99
|
+
)
|
|
100
|
+
return text
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def substitute_epoch(text: str, epoch: str | None) -> str:
|
|
104
|
+
"""Resolve ``{{epoch}}`` to one installation's publication instant.
|
|
105
|
+
|
|
106
|
+
Separate from :func:`expand` because the value is not a destination's
|
|
107
|
+
business: one install writes Registry rows for several items against several
|
|
108
|
+
targets, and they all have to carry the same instant or two rows published
|
|
109
|
+
by one build would order against each other.
|
|
110
|
+
|
|
111
|
+
A statement carrying the token when no epoch was supplied is a fault worth
|
|
112
|
+
naming here — the alternative is ``expand`` reporting it as an unresolvable
|
|
113
|
+
name, which says nothing about the missing value.
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
if not EPOCH.search(text):
|
|
117
|
+
return text
|
|
118
|
+
if epoch is None:
|
|
119
|
+
raise InstallError(
|
|
120
|
+
"a statement names {{epoch}} but this installation supplied none, so "
|
|
121
|
+
"the row it writes could not be dated"
|
|
122
|
+
)
|
|
123
|
+
return EPOCH.sub(epoch.replace("\\", "\\\\"), text)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _part(value: str, what: str) -> str:
|
|
127
|
+
"""One name part, checked against the delimiters the token is built from."""
|
|
128
|
+
|
|
129
|
+
name = (value or "").strip()
|
|
130
|
+
if not name:
|
|
131
|
+
raise ValueError(f"{what} must not be empty")
|
|
132
|
+
for character in (".", "{", "}"):
|
|
133
|
+
if character in name:
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"{what} must not contain {character!r}, which delimits an "
|
|
136
|
+
f"object token: {value!r}"
|
|
137
|
+
)
|
|
138
|
+
return name
|