weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/executors/spark_table.py

@@ -0,0 +1,213 @@
+"""Spark SQL table build — inference and creation in one self-contained action.
+
+A Spark SQL table's shape is only settled by running its query in the session, so
+its payload is not finished SQL. It is a JSON instruction (built by
+:func:`weaver.ses.ddl._spark_table_ddl`) that this executor completes in a single
+pass, the Spark counterpart of the old T-SQL self-contained script
+(how-does-build-work §2):
+
+1. run the query and read the resulting ``DataFrame`` schema — Spark resolves the
+   column names and types from the logical plan without running a job, so no rows
+   are read;
+2. validate the columns with the same guards a declared schema passes at parse
+   (:func:`weaver.ses.columns.validate_build_columns`), driven entirely by the
+   frozen payload — the Weaver document source is never reopened;
+3. choose the physical business columns — declared types when declared, the
+   query's inferred types otherwise;
+4. append Weaver's audit columns;
+5. create the table with strict ``CREATE TABLE``.
+
+A Delta table has no identity column, so nothing here handles one. Native
+identity is what makes the column worth having, and no Delta version Weaver
+runs on generates it, so the ``Identity`` header is a Warehouse-only
+declaration the parser refuses elsewhere (:data:`weaver.declaration.metadata.IDENTITY_LANGUAGES`)
+rather than something accepted here and quietly not materialised.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ...errors import InstallError
+from ...declaration.columns import validate_build_columns
+from ...declaration.metadata import AUDIT_COLUMNS, audit_column_name, PYTHON
+from ...spark import tokens
+from ..models import BuildAction
+from .base import InstallationContext
+from .spark_case import exact_identifier_case
+
+#: Reserved audit names, in the Delta (underscored) spelling, for collision
+#: detection against an inferred query's own output columns.
+_AUDIT_NAMES = {audit_column_name(logical, PYTHON).lower() for logical in AUDIT_COLUMNS}
+
+class SparkTableExecutor:
+    name = "spark_table"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if payload is None:
+            raise InstallError(f"spark_table action {action.id!r} has no payload")
+        if context.spark is None:
+            raise InstallError(
+                f"spark_table action {action.id!r} needs a Spark session but none "
+                "was provided"
+            )
+
+        instruction = json.loads(payload.decode("utf-8"))
+        catalogue = context.catalogue
+        # Both sides are resolved against the batch's destination: the table this
+        # creates, and every managed object its query reads. Inferring the shape
+        # from a query that resolved through the session's own catalogue would
+        # read some other Lakehouse's table of that name — and then create a table
+        # of that shape, silently, in the right place.
+        qualified = catalogue.expand(instruction["object"])
+        query = catalogue.expand(instruction["source_query"])
+
+        # Fabric defaults case-sensitive analysis off. Weaver identities are exact,
+        # so the source query and the resulting DDL must share one exact-case scope:
+        # otherwise a table created as ``CustomerEnriched`` cannot be consumed by
+        # the next action in the same coordinated build.
+        with exact_identifier_case(
+            context.spark,
+            enabled=catalogue.destination.preserve_table_identifier_case,
+        ):
+            frame = context.spark.sql(query)
+            query_columns = tuple(field.name for field in frame.schema.fields)
+            query_types = {
+                field.name: field.dataType.simpleString() for field in frame.schema.fields
+            }
+
+            declared = instruction["declared_columns"]
+            declared_names = (
+                tuple(name for name, _type, _nn in declared)
+                if declared is not None
+                else None
+            )
+            references = tuple(
+                (label, column) for label, column in instruction["references"]
+            )
+            business_columns = validate_build_columns(
+                qualified,
+                query_columns,
+                declared_columns=declared_names,
+                references=references,
+            )
+
+            business = self._physical_columns(
+                qualified, business_columns, declared, query_types, references
+            )
+            physical = business + [
+                tuple(entry) for entry in instruction["audit_columns"]
+            ]
+
+            statement = _create_table_sql(
+                qualified,
+                physical,
+                column_mapping=instruction.get("column_mapping", True),
+            )
+            _create_preserving_identifier_case(
+                context.spark,
+                statement,
+                catalogue=catalogue,
+                logical_object=instruction["object"],
+            )
+        return {
+            "object": qualified,
+            "schema_mode": instruction["schema_mode"],
+            "columns": [name for name, _type, _nn in physical],
+        }
+
+    def _physical_columns(
+        self,
+        qualified: str,
+        business_columns: tuple[str, ...],
+        declared: list | None,
+        query_types: dict[str, str],
+        references: tuple[tuple[str, str], ...],
+    ) -> list[tuple[str, str, bool]]:
+        """The business columns as ``(name, type, not_null)``.
+
+        Declared columns carry their declared type and not-null. Inferred columns
+        take the query's type and are not null when the primary key or a
+        ``Not null`` names them — the same loading contract, applied to a shape
+        the query supplied rather than a declaration.
+        """
+
+        collisions = [name for name in business_columns if name.lower() in _AUDIT_NAMES]
+        if collisions:
+            raise InstallError(
+                f"{qualified}: the query produces column(s) reserved for Weaver's "
+                "audit columns: " + ", ".join(collisions)
+            )
+
+        if declared is not None:
+            declared_by_name = {name: (type_, nn) for name, type_, nn in declared}
+            return [(name, *declared_by_name[name]) for name in business_columns]
+
+        not_null_names = {
+            column for label, column in references if label in ("Primary key", "Not null")
+        }
+        return [
+            (name, query_types[name], name in not_null_names)
+            for name in business_columns
+        ]
+
+
+def _create_table_sql(
+    qualified: str, columns: list[tuple[str, str, bool]], *, column_mapping: bool
+) -> str:
+    column_lines = ",\n".join(
+        f"    {_ident(name)} {type_}{' NOT NULL' if not_null else ''}"
+        for name, type_, not_null in columns
+    )
+    mapping = (
+        "\nTBLPROPERTIES ('delta.columnMapping.mode' = 'name')" if column_mapping else ""
+    )
+    return (
+        f"CREATE TABLE {qualified} (\n"
+        f"{column_lines}\n"
+        ")\n"
+        "USING delta"
+        f"{mapping}\n"
+    )
+
+
+def _create_preserving_identifier_case(
+    spark,
+    statement: str,
+    *,
+    catalogue,
+    logical_object: str,
+) -> None:
+    """Create one exact-case table without leaking session configuration."""
+
+    spark.sql(statement)
+
+
+def _drop_case_variant(catalogue, logical_object: str) -> None:
+    match = tokens.OBJECT.fullmatch(logical_object)
+    if match is None:  # expand() reports the useful token error on the main path
+        return
+    schema, declared = match.groups()
+    matches = [
+        existing
+        for existing in catalogue.tables(schema)
+        if existing.casefold() == declared.casefold()
+    ]
+    if not matches or matches == [declared]:
+        return
+    if len(matches) != 1:
+        raise InstallError(
+            f"{schema}.{declared}: target contains case-colliding tables: "
+            + ", ".join(sorted(matches))
+        )
+    catalogue.spark.sql(f"DROP TABLE {catalogue.qualify(schema, matches[0])}")
+
+
+def _ident(name: str) -> str:
+    return "`" + name.replace("`", "``") + "`"

weaver/build_bundle/executors/sql_endpoint_refresh.py

@@ -0,0 +1,34 @@
+"""Refresh a target Lakehouse's SQL analytics endpoint where supported."""
+
+from __future__ import annotations
+
+from ...errors import InstallError
+from ..models import REFRESH_SQL_ENDPOINT
+from .base import InstallationContext, SkippedExecution
+
+
+class SqlEndpointRefreshExecutor:
+    """Perform the planned refresh, or explicitly skip an unsupported host."""
+
+    name = "sql_endpoint_refresh"
+
+    def execute(self, action, payload, context: InstallationContext):
+        if action.kind != REFRESH_SQL_ENDPOINT:
+            raise InstallError(
+                f"SQL endpoint refresh action {action.id!r} has unknown kind "
+                f"{action.kind!r}"
+            )
+        if payload is not None:
+            raise InstallError(
+                f"SQL endpoint refresh action {action.id!r} must not carry a payload"
+            )
+
+        refresh = getattr(context.resolver, "refresh_sql_endpoint", None)
+        if refresh is None:
+            return SkippedExecution(
+                {
+                    "reason": "SQL endpoint refresh is unsupported in this environment",
+                }
+            )
+        details = refresh(context.target.lakehouse)
+        return details or {"lakehouse": context.target.bound.name}

weaver/build_bundle/executors/tsql.py

@@ -0,0 +1,81 @@
+"""T-SQL execution — run a generated Warehouse script through the SQL stack.
+
+The payload is a finished, self-contained T-SQL script (built by
+:mod:`weaver.declaration.tsql_ddl`): a table build materialises and inspects its own
+query shape server-side and creates only its main table; a view is a
+strict ``CREATE VIEW``. The executor runs it as one multi-statement script
+through the pooled SQL executor the environment supplies — it adds no logic of
+its own, exactly the mechanical executor the build philosophy calls for.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ...errors import InstallError
+from ..models import BuildAction
+from .base import InstallationContext
+
+
+class TSqlExecutor:
+    name = "tsql"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if payload is None:
+            raise InstallError(f"tsql action {action.id!r} has no payload")
+        if context.sql is None:
+            raise InstallError(
+                f"tsql action {action.id!r} needs a SQL executor but none was "
+                "provided — a Warehouse install must supply one"
+            )
+        script = payload.decode("utf-8")
+        context.sql.execute_script(script)
+        return {
+            "statement_first_line": script.splitlines()[0] if script.strip() else ""
+        }
+
+
+class TSqlBatchExecutor:
+    """Ordered T-SQL statements, each run as its own batch, as one action.
+
+    The T-SQL twin of ``spark_sql_batch``, and it exists for a reason the Spark
+    side does not have: several statements cannot share a batch when any of them
+    is a ``CREATE VIEW``, because T-SQL requires that to be the first statement in
+    its batch. ``execute_script`` sends what it is given as one batch, so a script
+    holding two ``CREATE OR ALTER VIEW`` statements is rejected outright —
+    *Incorrect syntax near the keyword 'create'*.
+
+    So the payload is an ordered array rather than one script, and each element is
+    submitted on its own. The action stays one action: it is one decision, reported
+    once, and the batching is transport.
+    """
+
+    name = "tsql_batch"
+
+    def execute(
+        self,
+        action: BuildAction,
+        payload: bytes | None,
+        context: InstallationContext,
+    ) -> dict[str, Any] | None:
+        if payload is None:
+            raise InstallError(f"tsql_batch action {action.id!r} has no payload")
+        if context.sql is None:
+            raise InstallError(
+                f"tsql_batch action {action.id!r} needs a SQL executor but none was "
+                "provided — a Warehouse install must supply one"
+            )
+        statements = json.loads(payload.decode("utf-8"))
+        if not isinstance(statements, list):
+            raise InstallError(
+                f"tsql_batch action {action.id!r} payload must be an array of statements"
+            )
+        for statement in statements:
+            context.sql.execute_script(statement)
+        return {"statements": len(statements)}

weaver/build_bundle/incremental.py

@@ -0,0 +1,288 @@
+"""Pure incremental selection from a prepared repository and trusted catalogue."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Iterable, Mapping
+
+from ..catalogue.state import RegisteredDocument
+from ..declaration.model import WeaverDocumentId, WeaverItemId, WeaverRepository
+
+
+def _ordered(values: Iterable[WeaverDocumentId]) -> tuple[WeaverDocumentId, ...]:
+    return tuple(sorted(set(values), key=str))
+
+
+@dataclass(frozen=True)
+class Impact:
+    """Signature classification plus existing descendants affected by changes."""
+
+    new: tuple[WeaverDocumentId, ...]
+    changed: tuple[WeaverDocumentId, ...]
+    impacted_descendants: tuple[WeaverDocumentId, ...]
+
+    @property
+    def impacted(self) -> tuple[WeaverDocumentId, ...]:
+        return _ordered(set(self.changed) | set(self.impacted_descendants))
+
+    def to_mapping(self) -> dict[str, list[str]]:
+        return {
+            "new": [str(value) for value in self.new],
+            "changed": [str(value) for value in self.changed],
+            "impacted_descendants": [
+                str(value) for value in self.impacted_descendants
+            ],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping) -> "Impact":
+        changed = tuple(
+            WeaverDocumentId.parse(value) for value in mapping.get("changed", ())
+        )
+        descendants = mapping.get("impacted_descendants")
+        if descendants is None:
+            descendants = tuple(
+                value
+                for value in mapping.get("impacted", ())
+                if WeaverDocumentId.parse(value) not in set(changed)
+            )
+        return cls(
+            new=tuple(WeaverDocumentId.parse(value) for value in mapping.get("new", ())),
+            changed=changed,
+            impacted_descendants=tuple(
+                WeaverDocumentId.parse(value) for value in descendants
+            ),
+        )
+
+
+@dataclass(frozen=True)
+class BuildSelection:
+    """The complete, inspectable decision before bundle actions are rendered."""
+
+    impact: Impact
+    prohibited: tuple[WeaverDocumentId, ...]
+    selected_for_drop: tuple[WeaverDocumentId, ...]
+    selected_for_build: tuple[WeaverDocumentId, ...]
+
+    def to_mapping(self) -> dict:
+        return {
+            "impact": self.impact.to_mapping(),
+            "prohibited": [str(value) for value in self.prohibited],
+            "selected_for_drop": [str(value) for value in self.selected_for_drop],
+            "selected_for_build": [str(value) for value in self.selected_for_build],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping) -> "BuildSelection":
+        return cls(
+            impact=Impact.from_mapping(mapping.get("impact", {})),
+            prohibited=tuple(
+                WeaverDocumentId.parse(value)
+                for value in mapping.get("prohibited", ())
+            ),
+            selected_for_drop=tuple(
+                WeaverDocumentId.parse(value)
+                for value in mapping.get("selected_for_drop", ())
+            ),
+            selected_for_build=tuple(
+                WeaverDocumentId.parse(value)
+                for value in mapping.get("selected_for_build", ())
+            ),
+        )
+
+
+def _as_instant(value) -> datetime | None:
+    """One build epoch as something comparable, whatever the reader returned.
+
+    Spark hands back a ``datetime``; a hand-built catalogue or a JSON round trip
+    may hand back the string that was written. Anything else is treated as no
+    epoch at all rather than guessed at — a wrong comparison here would rebuild
+    the estate or fail to.
+    """
+
+    if isinstance(value, datetime):
+        return value
+    if isinstance(value, str) and value:
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            return None
+    return None
+
+
+def stale_alias_destinations(
+    repository: WeaverRepository,
+    registered: Mapping[WeaverDocumentId, RegisteredDocument],
+    *,
+    bound_items: Iterable[WeaverItemId],
+) -> tuple[WeaverDocumentId, ...]:
+    """Aliases whose source has been rebuilt since the alias was last published.
+
+    This is the half of cross-item freshness the graph cannot answer, and it is
+    needed *whether or not* the producer is in this build. The descendant walk
+    only carries impact from a producer whose declaration changed; a producer
+    rebuilt by some earlier build is, to this one, entirely unchanged. Nothing in
+    the repository records that it moved — the only surviving evidence is in the
+    catalogue.
+
+    So the catalogue is asked directly: the producer's Registry row and the
+    alias's Registry row each carry the build that published them, and a producer
+    published later than the alias over it means the alias's consumers were built
+    against something that has since moved on. Naming it here lets it join the
+    ordinary changed roots, and its consumers are picked up by the ordinary walk.
+
+    ``bound_items`` scopes it to aliases this build could act on. A consumer item
+    that is not being built keeps its stale alias — that is the deferral, and it
+    is why the comparison is worth recording rather than acting on immediately.
+
+    Deliberately silent when either row is absent: that is not staleness but a
+    missing installation, which signature classification already calls new.
+    """
+
+    bound = set(bound_items)
+    stale = []
+    for alias in repository.aliases:
+        if alias.destination.item not in bound:
+            continue
+        destination = registered.get(alias.destination)
+        source = registered.get(alias.source)
+        if destination is None or source is None:
+            continue
+        source_epoch = _as_instant(source.build_epoch)
+        if source_epoch is None:
+            continue
+        destination_epoch = _as_instant(destination.build_epoch)
+        if destination_epoch is None or source_epoch > destination_epoch:
+            stale.append(alias.destination)
+    return _ordered(stale)
+
+
+def declared_signatures(
+    repository: WeaverRepository,
+    selected: Iterable[WeaverDocumentId],
+) -> dict[WeaverDocumentId, str]:
+    """What each selected node's Registry signature should be, from the source.
+
+    Three kinds of node are selectable and they are signed differently. A
+    document is signed by its source file. An alias destination is signed by the
+    pair it declares — this destination, that source — because that is the whole
+    of what an alias *is* (see
+    :attr:`~weaver.declaration.model.RepositoryAlias.signature`). A load artefact
+    is signed by what it is rendered from: a deployed module by its own bytes, a
+    generated body by the document it renders plus the version of the generator
+    that rendered it (see :mod:`weaver.etl`).
+    """
+
+    from ..etl import load_artefacts, load_artefacts_by_identity
+
+    aliases = {alias.destination: alias for alias in repository.aliases}
+    loads = load_artefacts_by_identity(load_artefacts(repository))
+    signatures: dict[WeaverDocumentId, str] = {}
+    for identity in selected:
+        alias = aliases.get(identity)
+        artefact = loads.get(identity)
+        if alias is not None:
+            signatures[identity] = alias.signature
+        elif artefact is not None:
+            signatures[identity] = artefact.signature
+        else:
+            signatures[identity] = repository.source_documents[
+                identity
+            ].effective_signature
+    return signatures
+
+
+def determine_impact(
+    repository: WeaverRepository,
+    registered: Mapping[WeaverDocumentId, RegisteredDocument],
+    *,
+    selected: Iterable[WeaverDocumentId],
+    stale_aliases: Iterable[WeaverDocumentId] = (),
+) -> Impact:
+    """Classify bound nodes and expand changed roots across the whole graph.
+
+    Propagation is no longer confined to one item. The graph carries alias
+    destinations as nodes, so the path from a producer to another item's consumer
+    is an ordinary walk — ``source → alias destination → consumer`` — and a
+    coordinated bundle that binds both items propagates across it exactly as it
+    does within one. Items *not* in the build are still deferred, but by
+    construction rather than by rule: they are not in ``selected``, so nothing
+    reaches them.
+
+    ``stale_aliases`` are destinations the catalogue already proved out of date —
+    their source was rebuilt by some earlier build that did not include this item
+    (see :func:`weaver.build_bundle.workflow.stale_alias_destinations`). They join
+    the changed roots, so their consumers are picked up by the same walk.
+    """
+
+    selected_set = set(selected)
+    installed = {
+        identity: document.signature
+        for identity, document in registered.items()
+        if identity in selected_set
+    }
+    declared = declared_signatures(repository, selected_set)
+
+    new: set[WeaverDocumentId] = set()
+    changed: set[WeaverDocumentId] = set()
+    for identity in selected_set:
+        signature = installed.get(identity)
+        if signature is None:
+            new.add(identity)
+        elif signature != declared[identity]:
+            changed.add(identity)
+    changed |= {
+        identity for identity in stale_aliases if identity in installed
+    }
+
+    existing = set(installed)
+    impacted = set(changed)
+    graph = repository.dependency_graph
+    if graph is not None:
+        by_text = {str(identity): identity for identity in selected_set}
+        for root in changed:
+            # A load artefact is not a node in the authored graph, and that is
+            # the design rather than an omission: nothing depends on a deployed
+            # module, and it depends on nothing — its signature is its own
+            # content. So a changed one is the end of a walk, not the start.
+            if root.is_load_artefact:
+                continue
+            for node in graph.descendants(str(root)):
+                descendant = by_text.get(node)
+                if descendant is not None and descendant in existing:
+                    impacted.add(descendant)
+
+    return Impact(
+        new=_ordered(new),
+        changed=_ordered(changed),
+        impacted_descendants=_ordered(impacted - changed),
+    )
+
+
+def select_build(
+    repository: WeaverRepository,
+    registered: Mapping[WeaverDocumentId, RegisteredDocument],
+    *,
+    selected: Iterable[WeaverDocumentId],
+    stale_aliases: Iterable[WeaverDocumentId] = (),
+) -> BuildSelection:
+    impact = determine_impact(
+        repository, registered, selected=selected, stale_aliases=stale_aliases
+    )
+    # An alias destination has no source document and therefore no
+    # ``prohibit_rebuild``: nothing an author writes can forbid replacing a
+    # pointer, because replacing one destroys nothing.
+    prohibited = {
+        identity
+        for identity in impact.impacted
+        if identity in repository.source_documents
+        and repository.source_documents[identity].document.prohibit_rebuild
+    }
+    selected_for_drop = set(impact.impacted) - prohibited
+    return BuildSelection(
+        impact=impact,
+        prohibited=_ordered(prohibited),
+        selected_for_drop=_ordered(selected_for_drop),
+        selected_for_build=_ordered(set(impact.new) | selected_for_drop),
+    )