weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/executors/folder.py

@@ -0,0 +1,71 @@
+"""Folder execution — strictly create/drop a managed folder, or prune one.
+
+Building a Folder is creating its directory in the Lakehouse Files area; there is
+no data (staging files into it is *load*). Pruning one is removing a directory
+the build already decided, at freeze time, is unmanaged. Both resolve their path
+from the action's resource id and the bound target and touch no catalog:
+
+- ``build_folder``  make ``Files/<schema>/<object>`` and fail on collision;
+- ``drop_folder``   remove a selected managed object and fail if absent;
+- ``prune_folder``  remove ``Files/<schema>/<object>`` (an object), or
+  ``Files/<schema>`` when the whole schema is unmanaged (resource ``folder:<schema>``).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ...errors import InstallError
+from ...targets import FolderTarget
+from ..models import BUILD_FOLDER, DROP_FOLDER, PRUNE_FOLDER, BuildAction
+from .base import InstallationContext
+
+
+class FolderExecutor:
+    name = "folder"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if action.resource_node_id is None:
+            raise InstallError(f"folder action {action.id!r} names no resource")
+        location = self._location(action.resource_node_id, context)
+        if action.kind == BUILD_FOLDER:
+            if context.store.exists(location):
+                raise InstallError(
+                    f"cannot create managed folder because it already exists: "
+                    f"{location.value}"
+                )
+            context.store.make_directory(location)
+            return {"created": location.value}
+        if action.kind == DROP_FOLDER:
+            if not context.store.exists(location):
+                raise InstallError(
+                    f"cannot drop managed folder because it does not exist: "
+                    f"{location.value}"
+                )
+            context.store.delete(location, recursive=True)
+            return {"dropped": location.value}
+        if action.kind == PRUNE_FOLDER:
+            if context.store.exists(location):
+                context.store.delete(location, recursive=True)
+            return {"pruned": location.value}
+        raise InstallError(f"folder action {action.id!r} has unknown kind {action.kind!r}")
+
+    def _location(self, node_id: str, context: InstallationContext):
+        target = FolderTarget(lakehouse=context.target.lakehouse)
+        if "/Files/" in node_id:
+            # Item-oriented canonical identity. The batch already carries the
+            # physical Lakehouse binding, so the logical item prefix is only
+            # identity and is not reinterpreted here.
+            qualified = node_id.split("/Files/", 1)[1]
+        else:
+            qualified = node_id.split(":", 1)[1]
+        if "." in qualified:  # a specific folder object
+            schema, name = qualified.split(".", 1)
+            return context.resolver.folder_object(target, schema, name)
+        # a whole unmanaged folder schema: the schema directory itself
+        return context.resolver.files_root(context.target.lakehouse).join(qualified)

weaver/build_bundle/executors/load_file.py

@@ -0,0 +1,205 @@
+"""Writing and removing one file of a Lakehouse item's deployed runtime tree.
+
+The load layer's file half — and for one kind of file, its second phase.
+
+A ``write_file`` action usually carries the exact bytes to put down: a deployed
+Python module is authored source and travels verbatim. A **generated Spark SQL
+load does not**. Its payload is an instruction, and finishing it is this
+executor's job:
+
+.. code-block:: text
+
+    read the built target's schema, through Spark
+    take its physical business columns
+    render the executable program from the instruction
+    resolve the destination tokens
+    write the file
+
+That two-phase shape is the same one the Warehouse load uses, which ships a
+script that reads ``sys.columns`` and assembles the procedure server-side. The
+reason is the same too: what a load writes are the *physical* target's columns,
+and a Spark SQL table may leave its schema to be inferred at build — so the
+program cannot be finished while the table is still a declaration. Writing a
+file up front would be writing down a guess.
+
+A consequence worth stating: installing a generated load therefore **needs a
+Spark session**, where deploying a module needs only the store.
+
+A ``delete_file`` action removes a file whose source has stopped claiming it.
+
+Both derive their location the same way every other executor does: from the
+action's resource id and the target the batch names. The identity says where the
+file goes (``_/Load/lib/dates.py`` beneath ``Files``) and the bound target says
+which Lakehouse, so nothing here decides placement — that was settled when the
+artefact was claimed.
+
+Directories are the store's business on the way down, and nobody's on the way
+back up. The tree is owned by a declared folder, so when the last artefact goes
+the folder stops being projected and ordinary folder prune removes the whole
+subtree — an executor walking upward deleting empty parents would be a second,
+quieter answer to a question already answered.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ...declaration.load import TSQL_LOAD_EXTENSION as SQL_EXTENSION
+from ...declaration.spark_load import GENERATED_LOAD_MARKER
+from ...spark import tokens
+
+
+def _generated_load(text: str) -> dict | None:
+    """The instruction this payload carries, or ``None`` if it is not one."""
+
+    import json
+
+    from ...declaration.spark_load import GENERATED_LOAD_INSTRUCTION
+
+    stripped = text.lstrip()
+    if not stripped.startswith("{"):
+        return None
+    try:
+        payload = json.loads(stripped)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(payload, dict):
+        return None
+    return payload if payload.get("weaver") == GENERATED_LOAD_INSTRUCTION else None
+from ...errors import InstallError
+from ...targets import FolderTarget
+from ..models import DELETE_FILE, WRITE_FILE, BuildAction
+from .base import InstallationContext
+
+
+class LoadFileExecutor:
+    name = "load_file"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if action.resource_node_id is None:
+            raise InstallError(f"load file action {action.id!r} names no resource")
+        location = self._location(action.resource_node_id, context)
+        if action.kind == WRITE_FILE:
+            if payload is None:
+                raise InstallError(f"load file action {action.id!r} has no payload")
+            payload = self._addressed(location.value, payload, context)
+            context.store.write(location, payload)
+            return {"written": location.value, "bytes": len(payload)}
+        if action.kind == DELETE_FILE:
+            # Tolerant of absence, and only here. A delete is reconciliation
+            # toward "this must not exist", and something else having already
+            # removed it is that state reached — unlike a create, where a
+            # collision means two things believe they own one name.
+            if context.store.exists(location):
+                context.store.delete(location)
+                return {"deleted": location.value}
+            return {"absent": location.value}
+        raise InstallError(
+            f"load file action {action.id!r} has unknown kind {action.kind!r}"
+        )
+
+    def _addressed(
+        self, path: str, payload: bytes, context: InstallationContext
+    ) -> bytes:
+        """Finish a generated load, and address it, on the way down.
+
+        Two steps, not one. An instruction is first *rendered* into a program
+        against the built target's columns (:meth:`_render`); whatever program
+        results then has its object tokens resolved. A payload that is already a
+        program — one written by an older bundle — skips straight to the second.
+
+        The bundle stays destination-free, which is what lets one repository
+        generate the same bytes everywhere and two bundles be diffed for what
+        actually differs. The *installed file* cannot be: it has to be runnable
+        by anyone who opens it, and by then the destination is known, so this is
+        the moment the two requirements stop conflicting.
+
+        Deployed Python is left exactly as authored. A module is source code, not
+        a statement, and it addresses its target through the resolved Lakehouse
+        it is constructed with.
+
+        **Decided by what the payload is, not by what it is called.** Keying on a
+        ``.spark.sql`` suffix looked right and was wrong: a generated load keeps
+        its *authored* name, ``Sales.OrderSummary.sql``, so the suffix never
+        matched and every installed program shipped with its tokens intact and
+        could not run. A generated Spark program announces itself in its first
+        line, which is a fact about the file rather than about its name.
+        """
+
+        if not path.endswith(SQL_EXTENSION):
+            return payload
+        text = payload.decode("utf-8")
+        instruction = _generated_load(text)
+        if instruction is not None:
+            text = self._render(instruction, context)
+        elif not text.lstrip().startswith(GENERATED_LOAD_MARKER):
+            return payload
+        destination = context.target.destination
+        if destination is None:
+            raise InstallError(
+                f"a generated load lands in {context.target.bound.id!r}, which "
+                "resolved to no Spark destination, so its object names cannot be "
+                "addressed"
+            )
+        # Resolved against the destination directly rather than through the
+        # catalogue: writing a file needs no Spark session, and asking for one
+        # would make installing a load depend on a capability it never uses.
+        return tokens.expand(text, destination).encode("utf-8")
+
+    def _render(self, instruction: dict, context: InstallationContext) -> str:
+        """Finish a generated load from the table it will write to.
+
+        The columns are the one thing generation cannot know: a Spark SQL table
+        may leave its schema to be inferred at build, and even a declared one is
+        materialised as the physical table the program has to name. So the
+        bundle carries an instruction and this reads the built table — the same
+        two-phase shape the Warehouse load uses with ``sys.columns``, rather
+        than writing a file from a guess and hoping the guess held.
+        """
+
+        from ...declaration.spark_load import render_installed_program
+        # The module rather than the name. `tests/test_core_boundary.py` matches
+        # raw source text, so pulling in a symbol whose name begins with the
+        # engine's reads to it as the forbidden dependency.
+        from ...runtime import load_contract
+
+        if context.spark is None:
+            raise InstallError(
+                f"a generated load for {instruction['qualified']} must read its "
+                "target's columns, and no Spark session was provided"
+            )
+        target = context.catalogue.expand(instruction["object"])
+        audit = set(load_contract.delta_audit_columns())
+        columns = tuple(
+            field.name
+            for field in context.spark.table(target).schema.fields
+            if field.name not in audit
+        )
+        if not columns:
+            raise InstallError(
+                f"{target} has no loadable columns; build the table before "
+                "installing its load"
+            )
+        return render_installed_program(instruction, columns)
+
+    def _location(self, node_id: str, context: InstallationContext):
+        """``Lakehouse/Sales/file:_/Load/lib/dates.py`` under this batch's target.
+
+        The logical item prefix is identity only: the batch already carries the
+        physical binding, so the path beneath ``Files`` is what is used and the
+        item it names is not reinterpreted here.
+        """
+
+        marker = "/file:"
+        if marker not in node_id:
+            raise InstallError(
+                f"load file action names {node_id!r}, which is not a file identity"
+            )
+        relative = node_id.split(marker, 1)[1]
+        target = FolderTarget(lakehouse=context.target.lakehouse)
+        return context.resolver.folder_root(target).join(*relative.split("/"))

weaver/build_bundle/executors/spark_case.py

@@ -0,0 +1,26 @@
+"""Exact-case Spark analysis scoped to one executor call."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Iterator
+
+_CASE_SENSITIVE = "spark.sql.caseSensitive"
+
+
+@contextmanager
+def exact_identifier_case(spark, *, enabled: bool) -> Iterator[None]:
+    """Temporarily make both analysis and DDL honour Weaver identifier case."""
+
+    if not enabled:
+        yield
+        return
+    previous = spark.conf.get(_CASE_SENSITIVE)
+    if str(previous).lower() == "true":
+        yield
+        return
+    spark.conf.set(_CASE_SENSITIVE, "true")
+    try:
+        yield
+    finally:
+        spark.conf.set(_CASE_SENSITIVE, previous)

weaver/build_bundle/executors/spark_schema.py

@@ -0,0 +1,60 @@
+"""Creating one schema in the destination the batch names.
+
+A schema create is the one piece of build DDL that cannot be a frozen SQL
+payload, and the reason is instructive: on local Spark it needs a ``LOCATION``,
+and a ``LOCATION`` is a *resolved path*.
+
+Freezing it meant a bundle generated on a laptop carried
+
+.. code-block:: sql
+
+    CREATE SCHEMA IF NOT EXISTS `Sales` LOCATION '/var/folders/…/T/pytest-42/Sales_LH/Tables/Sales'
+
+— a temporary directory, in the hashed plan, deciding where a managed table
+lands. Two runs of the same repository produced different bundles, and a bundle
+kept overnight named a path that no longer existed (how-does-build-work §15). On
+Fabric it froze the opposite mistake: no clause at all, and a bare two-part name,
+so the schema was created in whatever Lakehouse the session was attached to
+rather than in the destination.
+
+So the action names the schema and nothing else, and the destination decides how
+to make one. That is not an installer filling in a semantic decision — which
+schema, in which Lakehouse, is settled and in the manifest — it is the same
+transport-level resolution every other action gets, applied to a clause that is
+purely about storage.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ...errors import InstallError
+from ..models import BuildAction
+from .base import InstallationContext
+
+
+class SparkSchemaExecutor:
+    name = "spark_schema"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if payload is None:
+            raise InstallError(f"spark_schema action {action.id!r} has no payload")
+        if context.spark is None:
+            raise InstallError(
+                f"spark_schema action {action.id!r} needs a Spark session but none "
+                "was provided"
+            )
+        schema = json.loads(payload.decode("utf-8"))["schema"]
+        catalogue = context.catalogue
+        statement = catalogue.create_schema(schema, if_not_exists=False)
+        return {
+            "destination": catalogue.destination.item,
+            "schema": catalogue.qualified_schema(schema),
+            "statement": statement,
+        }

weaver/build_bundle/executors/spark_sql.py

@@ -0,0 +1,59 @@
+"""Spark SQL execution — run the generated statement against the batch's target.
+
+The payload is the single executable unit ``create_ddl`` produced: a ``CREATE OR
+REPLACE VIEW``/``TABLE``, or a frozen prune ``DROP``. It names its objects
+logically — ``{{object:Sales.Customer}}`` — and this resolves those names against
+the destination the batch is bound to before running the statement.
+
+That resolution is the whole difference between a build that works and one that
+looks like it does. A two-part name resolves through the session's *current*
+catalogue, and the session is attached to the Weaver Lakehouse, so every
+destination statement would have landed in the control plane. On Fabric the
+object would have been created in the wrong Lakehouse and then read back from the
+wrong Lakehouse, and the assertion would have passed.
+
+The same session runs every sequence, so a view registered earlier is in the
+catalogue for a later one — now under a name that says which Lakehouse it is in.
+The SQL analytics endpoint is never used; Spark views are Spark-catalogue objects
+and resolve there.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ...errors import InstallError
+from ..models import BuildAction
+from .base import InstallationContext
+from .spark_case import exact_identifier_case
+
+
+class SparkSqlExecutor:
+    name = "spark_sql"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if payload is None:
+            raise InstallError(f"spark_sql action {action.id!r} has no payload")
+        if context.spark is None:
+            raise InstallError(
+                f"spark_sql action {action.id!r} needs a Spark session but none was provided"
+            )
+        catalogue = context.catalogue
+        statement = catalogue.expand(payload.decode("utf-8").strip())
+        with exact_identifier_case(
+            context.spark,
+            enabled=catalogue.destination.preserve_table_identifier_case,
+        ):
+            context.spark.sql(statement)
+        # The destination is reported, not just used: an install report that says
+        # which Lakehouse each statement ran against is the record a reviewer needs
+        # when the answer used to depend on what the session was attached to.
+        return {
+            "destination": catalogue.destination.item,
+            "statement_first_line": statement.splitlines()[0] if statement else "",
+        }

weaver/build_bundle/executors/spark_sql_batch.py

@@ -0,0 +1,57 @@
+"""Execute one ordered batch of Spark SQL statements from a JSON payload."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ...errors import InstallError
+from ...spark.tokens import substitute_epoch
+from ..models import BuildAction
+from .base import InstallationContext
+from .spark_case import exact_identifier_case
+
+
+class SparkSqlBatchExecutor:
+    name = "spark_sql_batch"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any]:
+        if payload is None:
+            raise InstallError(f"spark_sql_batch action {action.id!r} has no payload")
+        if context.spark is None:
+            raise InstallError(
+                f"spark_sql_batch action {action.id!r} needs a Spark session"
+            )
+        try:
+            statements = json.loads(payload.decode("utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise InstallError(
+                f"spark_sql_batch action {action.id!r} has an invalid JSON payload"
+            ) from exc
+        if not isinstance(statements, list) or not all(
+            isinstance(statement, str) and statement.strip()
+            for statement in statements
+        ):
+            raise InstallError(
+                f"spark_sql_batch action {action.id!r} must contain SQL strings"
+            )
+        with exact_identifier_case(
+            context.spark,
+            enabled=context.catalogue.destination.preserve_table_identifier_case,
+        ):
+            for statement in statements:
+                # The epoch first: it is scoped to this installation rather than
+                # to a destination, and ``expand`` rejects every token it does
+                # not itself resolve — so one left behind here would be reported
+                # as an unresolvable name instead of quietly reaching the engine.
+                dated = substitute_epoch(statement.strip(), context.epoch)
+                context.spark.sql(context.catalogue.expand(dated))
+        return {
+            "destination": context.catalogue.destination.item,
+            "statement_count": len(statements),
+        }