weaverstack 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. weaver/__init__.py +59 -0
  2. weaver/build_bundle/__init__.py +109 -0
  3. weaver/build_bundle/aliases.py +325 -0
  4. weaver/build_bundle/bundle.py +359 -0
  5. weaver/build_bundle/catalogue_actions.py +275 -0
  6. weaver/build_bundle/changes.py +186 -0
  7. weaver/build_bundle/endpoints.py +83 -0
  8. weaver/build_bundle/executors/__init__.py +69 -0
  9. weaver/build_bundle/executors/alias.py +202 -0
  10. weaver/build_bundle/executors/base.py +132 -0
  11. weaver/build_bundle/executors/folder.py +71 -0
  12. weaver/build_bundle/executors/load_file.py +205 -0
  13. weaver/build_bundle/executors/spark_case.py +26 -0
  14. weaver/build_bundle/executors/spark_schema.py +60 -0
  15. weaver/build_bundle/executors/spark_sql.py +59 -0
  16. weaver/build_bundle/executors/spark_sql_batch.py +57 -0
  17. weaver/build_bundle/executors/spark_table.py +213 -0
  18. weaver/build_bundle/executors/sql_endpoint_refresh.py +34 -0
  19. weaver/build_bundle/executors/tsql.py +81 -0
  20. weaver/build_bundle/incremental.py +288 -0
  21. weaver/build_bundle/installer.py +384 -0
  22. weaver/build_bundle/models.py +288 -0
  23. weaver/build_bundle/payloads.py +34 -0
  24. weaver/build_bundle/physical.py +625 -0
  25. weaver/build_bundle/planner.py +389 -0
  26. weaver/build_bundle/prune.py +620 -0
  27. weaver/build_bundle/report.py +108 -0
  28. weaver/build_bundle/stages.py +196 -0
  29. weaver/build_bundle/targets.py +272 -0
  30. weaver/build_bundle/workflow.py +585 -0
  31. weaver/catalogue/__init__.py +73 -0
  32. weaver/catalogue/builtin.py +238 -0
  33. weaver/catalogue/claims.py +121 -0
  34. weaver/catalogue/projection.py +437 -0
  35. weaver/catalogue/reader.py +152 -0
  36. weaver/catalogue/reconcile.py +231 -0
  37. weaver/catalogue/render.py +410 -0
  38. weaver/catalogue/state.py +660 -0
  39. weaver/catalogue/tables.py +648 -0
  40. weaver/config.py +178 -0
  41. weaver/declaration/__init__.py +171 -0
  42. weaver/declaration/columns.py +223 -0
  43. weaver/declaration/ddl.py +266 -0
  44. weaver/declaration/dependencies.py +544 -0
  45. weaver/declaration/graph.py +240 -0
  46. weaver/declaration/item_dependencies.py +292 -0
  47. weaver/declaration/load.py +191 -0
  48. weaver/declaration/metadata.py +1405 -0
  49. weaver/declaration/model.py +448 -0
  50. weaver/declaration/references.py +294 -0
  51. weaver/declaration/repository.py +959 -0
  52. weaver/declaration/schemas.py +135 -0
  53. weaver/declaration/source.py +674 -0
  54. weaver/declaration/spark_load.py +759 -0
  55. weaver/declaration/sql_shaping.py +591 -0
  56. weaver/declaration/templates/ddl/declared_create_table.sql +64 -0
  57. weaver/declaration/templates/ddl/infer_create_table.sql +97 -0
  58. weaver/declaration/templates/ddl/metadata_column_validation.sql +30 -0
  59. weaver/declaration/templates/load/column_metadata.sql +40 -0
  60. weaver/declaration/templates/load/full_replace_body.sql +21 -0
  61. weaver/declaration/templates/load/install_load_procedure.sql +27 -0
  62. weaver/declaration/templates/load/load_procedure.sql +48 -0
  63. weaver/declaration/templates/load/primary_key_body.sql +113 -0
  64. weaver/declaration/tsql_ddl.py +468 -0
  65. weaver/declaration/tsql_load.py +417 -0
  66. weaver/declaration/warehouse_type_mapping.yml +93 -0
  67. weaver/diagnostics.py +247 -0
  68. weaver/errors.py +61 -0
  69. weaver/etl.py +469 -0
  70. weaver/fabric/__init__.py +107 -0
  71. weaver/fabric/auth.py +137 -0
  72. weaver/fabric/capacity.py +143 -0
  73. weaver/fabric/client.py +147 -0
  74. weaver/fabric/environment.py +460 -0
  75. weaver/fabric/livy.py +478 -0
  76. weaver/fabric/notebooks.py +201 -0
  77. weaver/fabric/onelake.py +263 -0
  78. weaver/fabric/resolution.py +344 -0
  79. weaver/fabric/resources.py +245 -0
  80. weaver/fabric/session.py +148 -0
  81. weaver/fabric/shortcuts.py +120 -0
  82. weaver/fabric/sql.py +118 -0
  83. weaver/fabric/store.py +198 -0
  84. weaver/initialise.py +209 -0
  85. weaver/lakehouse.py +386 -0
  86. weaver/load.py +474 -0
  87. weaver/load_execution.py +483 -0
  88. weaver/load_plan.py +912 -0
  89. weaver/load_report.py +330 -0
  90. weaver/load_resolution.py +386 -0
  91. weaver/locations.py +164 -0
  92. weaver/objects.py +392 -0
  93. weaver/operations.py +757 -0
  94. weaver/physical_wipe.py +369 -0
  95. weaver/push.py +76 -0
  96. weaver/resolution.py +292 -0
  97. weaver/runtime/__init__.py +30 -0
  98. weaver/runtime/folder_load.py +402 -0
  99. weaver/runtime/load_contract.py +245 -0
  100. weaver/runtime/load_result.py +104 -0
  101. weaver/runtime/spark_load.py +152 -0
  102. weaver/runtime/table_load.py +497 -0
  103. weaver/spark/__init__.py +49 -0
  104. weaver/spark/catalogue.py +245 -0
  105. weaver/spark/destination.py +195 -0
  106. weaver/spark/session.py +84 -0
  107. weaver/spark/tokens.py +138 -0
  108. weaver/sql/__init__.py +40 -0
  109. weaver/sql/authentication.py +38 -0
  110. weaver/sql/connection.py +90 -0
  111. weaver/sql/errors.py +25 -0
  112. weaver/sql/execution.py +123 -0
  113. weaver/sql/pool.py +174 -0
  114. weaver/sql/wipe.py +156 -0
  115. weaver/store.py +209 -0
  116. weaver/targets.py +257 -0
  117. weaver/task_logging.py +215 -0
  118. weaver/unbind.py +74 -0
  119. weaver/workspaces.py +175 -0
  120. weaver_cli/__init__.py +12 -0
  121. weaver_cli/__main__.py +7 -0
  122. weaver_cli/main.py +626 -0
  123. weaverstack-0.1.1.dist-info/METADATA +113 -0
  124. weaverstack-0.1.1.dist-info/RECORD +127 -0
  125. weaverstack-0.1.1.dist-info/WHEEL +4 -0
  126. weaverstack-0.1.1.dist-info/entry_points.txt +2 -0
  127. weaverstack-0.1.1.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,245 @@
1
+ """Spark catalogue operations, against a *named* destination.
2
+
3
+ Everything Weaver does to a Lakehouse through Spark goes through here, and the
4
+ one thing this type refuses to do is assume the session is pointed at the right
5
+ place. The session is attached to the Weaver Lakehouse — the fixed control plane
6
+ — so a destination is never the current catalogue, and an operation that did not
7
+ name one would land in the control plane instead. That is the failure this
8
+ exists to make impossible, not merely unlikely.
9
+
10
+ A build has at least two of these open at once: one for the Lakehouse being
11
+ built, one for the Weaver Lakehouse the catalogue is written to. They differ only
12
+ in their :class:`~weaver.spark.destination.SparkDestination`, which is the claim
13
+ being made — that once a name is right, the operation is the same one.
14
+
15
+ **Enumerating a destination's schemas is not here, and that is deliberate.** On
16
+ Fabric it cannot be: a schema is a three-level name under ``spark_catalog``, and
17
+ ``SHOW SCHEMAS IN `workspace`.`lakehouse``` is refused — a bare ``SHOW SCHEMAS``
18
+ lists the *attached* Lakehouse and nothing else. Schema discovery therefore reads
19
+ the destination's ``Tables/`` area through the store, which works across
20
+ Lakehouses on both workspaces and is what prune already does. Offering a
21
+ ``list_schemas`` here that silently answered for the wrong Lakehouse would be the
22
+ ambient-context mistake wearing an abstraction (how-does-build-work §4).
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ from typing import Any
28
+
29
+ from ..errors import InstallError
30
+ from . import tokens
31
+ from .destination import SparkDestination
32
+
33
+
34
+ class SparkCatalogue:
35
+ """Catalogue operations against one logical Spark destination."""
36
+
37
+ def __init__(self, spark: Any, destination: SparkDestination) -> None:
38
+ if spark is None:
39
+ raise InstallError(
40
+ f"a Spark session is needed to reach {destination.item!r}, "
41
+ "and none was provided"
42
+ )
43
+ self.spark = spark
44
+ self.destination = destination
45
+ if destination.case_sensitive_analysis:
46
+ # Local's folded schema is lower-case and every declared object keeps
47
+ # its exact Weaver spelling. Unlike Fabric's catalogue, Spark's local
48
+ # session catalogue cannot look that object up again after reverting
49
+ # to case-insensitive analysis, so this is the emulator's session
50
+ # policy rather than a one-statement override.
51
+ spark.conf.set("spark.sql.caseSensitive", "true")
52
+
53
+ # --- naming -----------------------------------------------------------
54
+
55
+ def qualify(self, schema: str, name: str) -> str:
56
+ return self.destination.qualify(schema, name)
57
+
58
+ def qualified_schema(self, schema: str) -> str:
59
+ return self.destination.qualified_schema(schema)
60
+
61
+ def expand(self, statement: str) -> str:
62
+ """One payload's object tokens, resolved to this destination."""
63
+
64
+ return tokens.expand(statement, self.destination)
65
+
66
+ # --- execution ---------------------------------------------------------
67
+
68
+ def sql(self, statement: str) -> Any:
69
+ """Run one statement here, with its object tokens resolved first."""
70
+
71
+ return self.spark.sql(self.expand(statement))
72
+
73
+ # --- structure ---------------------------------------------------------
74
+
75
+ def create_schema(self, schema: str, *, if_not_exists: bool = True) -> str:
76
+ """Create a schema in this destination, and return the statement run.
77
+
78
+ The ``LOCATION`` clause is the destination's business, not the planner's:
79
+ local Spark needs one so a managed table lands under the Lakehouse's
80
+ ``Tables`` area, and a schema-enabled Fabric Lakehouse pins it natively and
81
+ must not be given one. It is also a resolved path, so it could not have
82
+ been frozen into a payload without tying the bundle to the machine that
83
+ generated it (how-does-build-work §15).
84
+ """
85
+
86
+ qualifier = " IF NOT EXISTS" if if_not_exists else ""
87
+ statement = f"CREATE SCHEMA{qualifier} {self.qualified_schema(schema)}"
88
+ location = self.destination.schema_location(schema)
89
+ if location is not None:
90
+ statement += f" LOCATION '{_escaped(location)}'"
91
+ self.spark.sql(statement)
92
+ return statement
93
+
94
+ def register_external_table(self, schema: str, name: str, location: str) -> str:
95
+ """Name a table in this destination whose storage it does not own.
96
+
97
+ This exists for one thing: an alias in the local emulator. Fabric
98
+ discovers a OneLake shortcut placed under a Lakehouse's ``Tables`` area by
99
+ itself, and the table simply appears in the catalogue; local Spark
100
+ discovers nothing, so the emulator has to say out loud what Fabric infers.
101
+
102
+ Unregistered first rather than created strictly, because an alias is a
103
+ pointer and re-pointing one is not a destructive transition. Dropping an
104
+ *external* table removes the registration and never the storage, which is
105
+ exactly the distinction that makes this safe: the data belongs to the item
106
+ that produced it.
107
+ """
108
+
109
+ qualified = self.qualify(schema, name)
110
+ self.spark.sql(f"DROP TABLE IF EXISTS {qualified}")
111
+ statement = (
112
+ f"CREATE TABLE {qualified} USING DELTA LOCATION '{_escaped(location)}'"
113
+ )
114
+ self.spark.sql(statement)
115
+ return statement
116
+
117
+ # --- discovery ---------------------------------------------------------
118
+
119
+ def schema_exists(self, schema: str) -> bool:
120
+ return bool(self.spark.catalog.databaseExists(self.qualified_schema(schema)))
121
+
122
+ def views(self, schema: str) -> tuple[str, ...]:
123
+ """Persistent view names in one schema of this destination.
124
+
125
+ Views are catalogue-only — there is no directory to find them in — so this
126
+ is the one part of an inventory that has to be asked of Spark.
127
+ """
128
+
129
+ rows = self._rows(f"SHOW VIEWS IN {self.qualified_schema(schema)}")
130
+ names = []
131
+ for row in rows:
132
+ data = row.asDict()
133
+ if data.get("isTemporary"):
134
+ continue
135
+ name = data.get("viewName") or data.get("name")
136
+ if name:
137
+ names.append(name)
138
+ return tuple(names)
139
+
140
+ def tables(self, schema: str) -> tuple[str, ...]:
141
+ """Table names in one schema of this destination.
142
+
143
+ ``SHOW TABLES`` returns views as well, so the views are taken back out.
144
+ Prune does not use this — a Delta table is a directory, and reading the
145
+ storage is what keeps reconciliation scoped to the one Lakehouse — but a
146
+ test asserting what a build actually created needs to ask the catalogue,
147
+ not the filesystem.
148
+ """
149
+
150
+ views = {name.lower() for name in self.views(schema)}
151
+ rows = self._rows(f"SHOW TABLES IN {self.qualified_schema(schema)}")
152
+ names = []
153
+ for row in rows:
154
+ data = row.asDict()
155
+ if data.get("isTemporary"):
156
+ continue
157
+ name = data.get("tableName") or data.get("name")
158
+ if name and name.lower() not in views:
159
+ names.append(name)
160
+ return tuple(names)
161
+
162
+ def exists(self, schema: str, name: str) -> bool:
163
+ """Whether one object exists in this destination, table or view."""
164
+
165
+ return bool(self.spark.catalog.tableExists(self.qualify(schema, name)))
166
+
167
+ def _rows(self, statement: str) -> list:
168
+ """Run a listing, reading an absent schema as an empty one.
169
+
170
+ A schema that is not there holds nothing, which is the answer an inventory
171
+ wants — and both workspaces raise for it rather than returning no rows. So the
172
+ absence is tolerated and everything else propagates, narrowly, for the same
173
+ reason :mod:`weaver.catalogue.reader` does it that way: a real failure read
174
+ as "nothing here" tells the next build that nothing is managed.
175
+ """
176
+
177
+ try:
178
+ return self.spark.sql(statement).collect()
179
+ except Exception as exception:
180
+ if _is_missing_schema(exception):
181
+ return []
182
+ raise
183
+
184
+
185
+ #: Spark's error class for a namespace that does not exist. A missing Lakehouse
186
+ #: reports the same one, which is what we want: an inventory of somewhere that is
187
+ #: not there is empty either way.
188
+ _ABSENT = frozenset({"SCHEMA_NOT_FOUND"})
189
+
190
+
191
+ def _is_missing_schema(exception: Exception) -> bool:
192
+ """Whether this means "not created yet" rather than "went wrong".
193
+
194
+ Keyed on Spark's error class, not on message text, so a reworded message
195
+ cannot quietly turn an infrastructure failure into an empty inventory. The
196
+ class name is consulted only when no error class is available — a stub session
197
+ in a test, or a connector that raises a plain error.
198
+ """
199
+
200
+ error_class = getattr(exception, "getErrorClass", None)
201
+ if callable(error_class):
202
+ try:
203
+ found = error_class()
204
+ except Exception: # pragma: no cover - a broken accessor is not absence
205
+ found = None
206
+ if found:
207
+ return found in _ABSENT
208
+ return any(name in str(exception) for name in _ABSENT) or (
209
+ "NoSuchNamespaceException" in type(exception).__name__
210
+ or "NoSuchDatabaseException" in type(exception).__name__
211
+ )
212
+
213
+
214
+ def _escaped(value: str) -> str:
215
+ return value.replace("\\", "\\\\").replace("'", "\\'")
216
+
217
+
218
+ def drop_local_destination_catalogue(
219
+ spark: Any, destination: SparkDestination
220
+ ) -> tuple[str, ...]:
221
+ """Forget every namespace folded beneath one emulated Lakehouse.
222
+
223
+ Local CLI sessions use a persistent metastore so a later process can see
224
+ what ``initialise`` and ``build`` registered. A local wipe must therefore
225
+ clear catalogue registrations as well as the Fabric-shaped filesystem tree;
226
+ Fabric performs that bookkeeping itself when its Lakehouse storage is
227
+ emptied and never calls this emulator-only primitive.
228
+ """
229
+
230
+ if destination.namespace or not destination.schema_prefix:
231
+ raise InstallError("local catalogue cleanup needs a folded local destination")
232
+ rows = spark.sql("SHOW DATABASES").collect()
233
+ prefix = destination.schema_prefix.casefold()
234
+ schemas = []
235
+ for row in rows:
236
+ data = row.asDict() if hasattr(row, "asDict") else {}
237
+ name = data.get("namespace") or data.get("databaseName") or data.get("schemaName")
238
+ if name and str(name).casefold().startswith(prefix):
239
+ schemas.append(str(name))
240
+ statements = []
241
+ for schema in sorted(schemas, key=str.casefold):
242
+ statement = f"DROP SCHEMA IF EXISTS {destination.qualified_schema(schema[len(destination.schema_prefix):])} CASCADE"
243
+ spark.sql(statement)
244
+ statements.append(statement)
245
+ return tuple(statements)
@@ -0,0 +1,195 @@
1
+ """How one logical Lakehouse is *named* in a Spark session.
2
+
3
+ :class:`~weaver.locations.LakehouseSparkLocation` answers where a destination's
4
+ bytes live. This answers the other half: what a statement has to write to reach
5
+ that destination's catalogue, given that the session is attached somewhere else.
6
+
7
+ The two workspaces disagree, and the disagreement is **data**, not behaviour — which
8
+ is why there is one class here and two constructors rather than two classes.
9
+
10
+ **Fabric** has a native namespace for exactly this. Under ``spark_catalog`` a
11
+ schema is a three-level name, so an object is four parts::
12
+
13
+ `Weaver`.`Play_Lakehouse_1`.`Sales`.`Customer`
14
+ ^workspace ^lakehouse ^schema ^object
15
+
16
+ One session can create, read and drop through that name in any Lakehouse in the
17
+ workspace, and can build a view in one over a table in another. Nothing has to be
18
+ attached, and nothing has to be switched.
19
+
20
+ **Local Spark** has no such namespace, and cannot be given one: a Delta catalogue
21
+ can only be the *session* catalogue (``DeltaCatalog`` extends
22
+ ``DelegatingCatalogExtension``, and registered as an ordinary named catalogue its
23
+ delegate is null), so ``spark.sql.catalog.<lakehouse>`` is not available. Its one
24
+ namespace level is the database. The proxy therefore folds the Lakehouse into
25
+ that level::
26
+
27
+ `sales_lh__sales`.`Customer`
28
+
29
+ which is not Fabric syntax and is not meant to be. What it reproduces is the
30
+ *property* Fabric's namespace provides and a bare ``Sales.Customer`` does not:
31
+ two destinations that declare a schema of the same name stay apart. Storage is
32
+ untouched by the folding — the database still carries an explicit ``LOCATION`` of
33
+ ``<lakehouse>/Tables/<schema>``, so a managed table lands exactly where the
34
+ Fabric layout puts it and the emulator keeps mirroring OneLake. The folded
35
+ database identifier is lower-case because the local session catalogue registers
36
+ it that way; declared object identifiers remain exact-case under the emulator's
37
+ case-sensitive analysis policy.
38
+
39
+ A destination is never carried in a build bundle. It is derived at install time
40
+ from the item the bundle names, because a Fabric namespace is workspace-specific
41
+ and a local one is rooted in a temporary directory — see
42
+ :class:`~weaver.locations.LakehouseSparkLocation` for why a bundle that moved
43
+ with either would stop being comparable between environments
44
+ (how-does-build-work §15).
45
+ """
46
+
47
+ from __future__ import annotations
48
+
49
+ import re
50
+ from dataclasses import dataclass
51
+
52
+ from ..errors import IdentityError
53
+
54
+ #: Local Spark accepts word characters in a database name and nothing else, so a
55
+ #: folded name is only legal if the Lakehouse's own name is.
56
+ _LEGAL_LOCAL_NAME = re.compile(r"\A\w+\Z")
57
+
58
+ #: What separates the Lakehouse from the schema in the folded local name. Two
59
+ #: characters, so a single underscore in either half cannot be mistaken for it.
60
+ LOCAL_SEPARATOR = "__"
61
+
62
+
63
+ def identifier(name: str) -> str:
64
+ """A back-tick quoted Spark identifier, safe for spaces and keywords."""
65
+
66
+ return "`" + name.replace("`", "``") + "`"
67
+
68
+
69
+ @dataclass(frozen=True)
70
+ class SparkDestination:
71
+ """One logical Lakehouse, as a Spark session addresses it.
72
+
73
+ ``item`` is the logical name, and is what appears in a message. ``namespace``
74
+ is whatever sits above the schema — the workspace and Lakehouse on Fabric,
75
+ nothing locally. ``schema_prefix`` is the local fold, empty on Fabric.
76
+
77
+ ``tables_root`` is present only for the platform that needs it: local Spark
78
+ drops a managed table into its own warehouse directory unless the database
79
+ says otherwise, so the proxy supplies a ``LOCATION``. A schema-enabled Fabric
80
+ Lakehouse pins its own, and answers None.
81
+ """
82
+
83
+ item: str
84
+ namespace: tuple[str, ...] = ()
85
+ schema_prefix: str = ""
86
+ tables_root: str | None = None
87
+ preserve_table_identifier_case: bool = False
88
+ lowercase_schema_identifier: bool = False
89
+ case_sensitive_analysis: bool = False
90
+
91
+ def schema_identifier(self, schema: str) -> str:
92
+ """The schema's name at its own namespace level, unquoted."""
93
+
94
+ name = f"{self.schema_prefix}{_checked(schema, what='schema')}"
95
+ return name.lower() if self.lowercase_schema_identifier else name
96
+
97
+ def qualified_schema(self, schema: str) -> str:
98
+ """The schema, fully qualified — what ``CREATE SCHEMA`` is given."""
99
+
100
+ return ".".join(
101
+ identifier(part)
102
+ for part in (*self.namespace, self.schema_identifier(schema))
103
+ )
104
+
105
+ def qualify(self, schema: str, name: str) -> str:
106
+ """One object, fully qualified — what every statement names it by."""
107
+
108
+ return (
109
+ f"{self.qualified_schema(schema)}"
110
+ f".{identifier(_checked(name, what='object name'))}"
111
+ )
112
+
113
+ def schema_location(self, schema: str) -> str | None:
114
+ """Where this destination's managed tables for a schema must be pinned.
115
+
116
+ None when the platform pins them itself, which is the Fabric answer and
117
+ the reason no path reaches a Fabric ``CREATE SCHEMA``.
118
+ """
119
+
120
+ if self.tables_root is None:
121
+ return None
122
+ return f"{self.tables_root.rstrip('/')}/{_checked(schema, what='schema')}"
123
+
124
+ def __str__(self) -> str:
125
+ return self.item if not self.namespace else ".".join(self.namespace)
126
+
127
+
128
+ def fabric_destination(*, workspace: str, lakehouse: str) -> SparkDestination:
129
+ """A Fabric Lakehouse, addressed by its native four-part name.
130
+
131
+ Both names are display names, deliberately: this is what Fabric's Spark
132
+ namespace is spelled with, and it is what a reviewer reading a statement can
133
+ recognise. The workspace and item *ids* stay where they belong — in
134
+ resolution, and in the bundle's target block.
135
+ """
136
+
137
+ return SparkDestination(
138
+ item=lakehouse,
139
+ namespace=(
140
+ _checked(workspace, what="workspace"),
141
+ _checked(lakehouse, what="lakehouse"),
142
+ ),
143
+ # Fabric otherwise folds a quoted table identifier to lower-case at
144
+ # creation.
145
+ preserve_table_identifier_case=True,
146
+ )
147
+
148
+
149
+ def local_destination(*, item: str, tables_root: str) -> SparkDestination:
150
+ """A local Lakehouse, folded into the one namespace level Spark offers.
151
+
152
+ The Lakehouse name becomes part of every database name, so it has to be a
153
+ legal one. Refused rather than sanitised: a silently altered name would make
154
+ two destinations collide again, which is the single thing this exists to
155
+ prevent.
156
+ """
157
+
158
+ name = _checked(item, what="lakehouse")
159
+ if not _LEGAL_LOCAL_NAME.match(name):
160
+ raise IdentityError(
161
+ f"local Spark folds the Lakehouse name into its database names, and "
162
+ f"only accepts letters, digits and underscores there — {item!r} cannot "
163
+ "be addressed locally"
164
+ )
165
+ return SparkDestination(
166
+ item=name,
167
+ schema_prefix=f"{name}{LOCAL_SEPARATOR}",
168
+ tables_root=tables_root,
169
+ # The emulator mirrors Fabric's case-preserving table directories. Its
170
+ # folded schema itself was registered under Spark's case-insensitive
171
+ # policy, so every statement addresses it by its canonical lower case.
172
+ preserve_table_identifier_case=True,
173
+ lowercase_schema_identifier=True,
174
+ # Unlike Fabric's catalogue, Spark's local session catalogue cannot find
175
+ # a PascalCase table again after analysis returns to case-insensitive
176
+ # mode. The emulator therefore uses one exact-case policy for its life.
177
+ case_sensitive_analysis=True,
178
+ )
179
+
180
+
181
+ def _checked(value: object, *, what: str) -> str:
182
+ """One name part, checked rather than trusted.
183
+
184
+ These strings are concatenated into identifiers and into paths, so a part
185
+ carrying a delimiter would name something other than what it says.
186
+ """
187
+
188
+ if not isinstance(value, str):
189
+ raise IdentityError(f"{what} must be a string, got {type(value).__name__}")
190
+ name = value.strip()
191
+ if not name:
192
+ raise IdentityError(f"{what} must not be empty")
193
+ if "." in name or "/" in name or "\\" in name:
194
+ raise IdentityError(f"{what} must not contain a separator: {value!r}")
195
+ return name
@@ -0,0 +1,84 @@
1
+ """A leak-free local Delta session for CLI execution."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import sys
7
+ from importlib import import_module
8
+ from contextlib import contextmanager
9
+ from pathlib import Path
10
+ from typing import Iterator
11
+
12
+ from ..diagnostics import SUPPORTED_JAVA, find_java_home
13
+ from ..errors import CommandError
14
+
15
+
16
+ @contextmanager
17
+ def local_delta_session(workspace=None) -> Iterator[object]:
18
+ """Create one local Delta session and always stop it before returning.
19
+
20
+ CLI invocations are separate JVMs. When a local Workspace is supplied its
21
+ Spark metastore therefore lives beneath that emulator root, so namespaces
22
+ and table registrations created by ``initialise`` remain visible to the next
23
+ ``build`` command just as Fabric's catalogue remains visible between
24
+ sessions. Tests that supply no Workspace keep Spark's process-local default.
25
+ """
26
+
27
+ try:
28
+ configure_spark_with_delta_pip = import_module(
29
+ "delta"
30
+ ).configure_spark_with_delta_pip
31
+ SparkSession = import_module("pyspark.sql").SparkSession
32
+ except ImportError as exc:
33
+ raise CommandError(
34
+ "local build needs the Spark extra; install weaverstack[spark]"
35
+ ) from exc
36
+
37
+ java_home = find_java_home()
38
+ if java_home is None:
39
+ raise CommandError(
40
+ f"local build needs Java {' or '.join(SUPPORTED_JAVA)}; run weaver doctor"
41
+ )
42
+ previous = {
43
+ name: os.environ.get(name)
44
+ for name in ("JAVA_HOME", "PYSPARK_PYTHON", "PYSPARK_DRIVER_PYTHON")
45
+ }
46
+ os.environ["JAVA_HOME"] = java_home
47
+ os.environ["PYSPARK_PYTHON"] = sys.executable
48
+ os.environ["PYSPARK_DRIVER_PYTHON"] = sys.executable
49
+ builder = (
50
+ SparkSession.builder.appName("weaverstack-cli")
51
+ .master("local[*]")
52
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
53
+ .config(
54
+ "spark.sql.catalog.spark_catalog",
55
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog",
56
+ )
57
+ .config("spark.ui.enabled", "false")
58
+ .config("spark.sql.shuffle.partitions", "1")
59
+ .config("spark.databricks.delta.snapshotPartitions", "1")
60
+ )
61
+ if workspace is not None:
62
+ root_value = getattr(workspace, "workspace", workspace)
63
+ root = Path(root_value).expanduser().resolve() / ".weaver" / "spark"
64
+ builder = (
65
+ builder.config("spark.sql.catalogImplementation", "hive")
66
+ .config("spark.sql.warehouse.dir", str(root / "warehouse"))
67
+ .config(
68
+ "javax.jdo.option.ConnectionURL",
69
+ f"jdbc:derby:;databaseName={root / 'metastore'};create=true",
70
+ )
71
+ )
72
+ session = None
73
+ try:
74
+ session = configure_spark_with_delta_pip(builder).getOrCreate()
75
+ session.sparkContext.setLogLevel("ERROR")
76
+ yield session
77
+ finally:
78
+ if session is not None:
79
+ session.stop()
80
+ for name, value in previous.items():
81
+ if value is None:
82
+ os.environ.pop(name, None)
83
+ else:
84
+ os.environ[name] = value
weaver/spark/tokens.py ADDED
@@ -0,0 +1,138 @@
1
+ """How a frozen payload names an object without naming a destination.
2
+
3
+ A generated statement has to say *which* object it acts on. It must not say
4
+ which Lakehouse, in which workspace, at which path: that is bound by the batch's
5
+ target, resolved at install time, and different in every environment. Writing it
6
+ into the SQL would make two bundles of the same repository differ in every
7
+ payload merely for having been generated somewhere else, which is exactly the
8
+ comparison how-does-build-work §15 exists to protect.
9
+
10
+ So a payload names an object logically, and the executor asks the batch's
11
+ destination what that is called there::
12
+
13
+ CREATE VIEW {{object:Sales.ActiveCustomer}} AS
14
+ SELECT * FROM {{object:Sales.Customer}} WHERE IsActive
15
+
16
+ Fabric -> `Weaver`.`Play_Lakehouse_1`.`Sales`.`ActiveCustomer`
17
+ local -> `sales_lh__sales`.`ActiveCustomer`
18
+
19
+ This is substitution of a transport-level value, not a template (§16). Nothing
20
+ semantic is left for the installer to decide: the object, its schema, the
21
+ statement and the destination are all fixed before the bundle is written — the
22
+ only thing supplied late is how that already-chosen destination spells a name.
23
+ A reviewer reading the payload sees the object; the manifest's target block says
24
+ where it goes. A bare two-part name said neither, and resolved through whatever
25
+ the session happened to be attached to.
26
+
27
+ The tokens are deliberately unmistakable. ``{{`` and ``}}`` are not Spark SQL, so
28
+ an unexpanded one is a syntax error at the point of use rather than a name that
29
+ quietly resolves somewhere else.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import re
35
+
36
+ from ..errors import InstallError
37
+ from .destination import SparkDestination
38
+
39
+ #: ``{{object:Schema.Name}}`` — one managed object.
40
+ OBJECT = re.compile(r"\{\{object:([^.{}]+)\.([^.{}]+)\}\}")
41
+
42
+ #: ``{{schema:Name}}`` — one managed schema.
43
+ SCHEMA = re.compile(r"\{\{schema:([^.{}]+)\}\}")
44
+
45
+ #: ``{{epoch}}`` — the instant this installation published its Registry.
46
+ #:
47
+ #: The one token that is not about a destination, which is why :func:`expand`
48
+ #: does not resolve it: it is scoped to the *installation*, and every statement
49
+ #: in a build must receive the same value however many destinations they name.
50
+ #: :func:`substitute_epoch` puts it in, and it has to run first — ``expand``
51
+ #: rejects any token it does not recognise, so an epoch that reached it would be
52
+ #: an error rather than silently surviving into the engine.
53
+ #:
54
+ #: It is a token rather than a literal frozen at generation time for the reason
55
+ #: this whole module exists: a rendered clock would make the same repository
56
+ #: produce different payload bytes on every run, and a bundle's identity is its
57
+ #: bytes.
58
+ EPOCH = re.compile(r"\{\{epoch\}\}")
59
+
60
+ #: The payload spelling of the publication epoch.
61
+ EPOCH_TOKEN = "{{epoch}}"
62
+
63
+ #: Anything else in token shape. Matched only so an unknown one is reported
64
+ #: rather than passed through to the engine as mystery syntax.
65
+ ANY = re.compile(r"\{\{[^{}]*\}\}")
66
+
67
+
68
+ def object_token(schema: str, name: str) -> str:
69
+ """The payload spelling of one object."""
70
+
71
+ return f"{{{{object:{_part(schema, 'schema')}.{_part(name, 'object name')}}}}}"
72
+
73
+
74
+ def schema_token(schema: str) -> str:
75
+ """The payload spelling of one schema."""
76
+
77
+ return f"{{{{schema:{_part(schema, 'schema')}}}}}"
78
+
79
+
80
+ def expand(text: str, destination: SparkDestination) -> str:
81
+ """Every token in ``text``, resolved against one destination.
82
+
83
+ An unrecognised token is an error. Leaving it in place would hand Spark
84
+ something it cannot parse — better — or, if the shape ever became valid
85
+ syntax, something that means the wrong thing — far worse.
86
+ """
87
+
88
+ text = OBJECT.sub(
89
+ lambda match: destination.qualify(match.group(1), match.group(2)), text
90
+ )
91
+ text = SCHEMA.sub(
92
+ lambda match: destination.qualified_schema(match.group(1)), text
93
+ )
94
+ leftover = ANY.search(text)
95
+ if leftover:
96
+ raise InstallError(
97
+ f"{leftover.group(0)} is not a name this installer knows how to "
98
+ f"resolve against {destination.item!r}"
99
+ )
100
+ return text
101
+
102
+
103
+ def substitute_epoch(text: str, epoch: str | None) -> str:
104
+ """Resolve ``{{epoch}}`` to one installation's publication instant.
105
+
106
+ Separate from :func:`expand` because the value is not a destination's
107
+ business: one install writes Registry rows for several items against several
108
+ targets, and they all have to carry the same instant or two rows published
109
+ by one build would order against each other.
110
+
111
+ A statement carrying the token when no epoch was supplied is a fault worth
112
+ naming here — the alternative is ``expand`` reporting it as an unresolvable
113
+ name, which says nothing about the missing value.
114
+ """
115
+
116
+ if not EPOCH.search(text):
117
+ return text
118
+ if epoch is None:
119
+ raise InstallError(
120
+ "a statement names {{epoch}} but this installation supplied none, so "
121
+ "the row it writes could not be dated"
122
+ )
123
+ return EPOCH.sub(epoch.replace("\\", "\\\\"), text)
124
+
125
+
126
+ def _part(value: str, what: str) -> str:
127
+ """One name part, checked against the delimiters the token is built from."""
128
+
129
+ name = (value or "").strip()
130
+ if not name:
131
+ raise ValueError(f"{what} must not be empty")
132
+ for character in (".", "{", "}"):
133
+ if character in name:
134
+ raise ValueError(
135
+ f"{what} must not contain {character!r}, which delimits an "
136
+ f"object token: {value!r}"
137
+ )
138
+ return name