weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/installer.py

@@ -0,0 +1,384 @@
+"""Installing a bundle — validated execution only, never planning.
+
+The installer loads and fully validates a bundle, resolves its targets through
+the supplied environment, and runs the sequences as barriers: each completes
+before the next starts, one action's failure fails its sequence, and no later
+sequence begins. It records exactly one result per action and persists the
+report. It never reads the source repository, resolves a dependency or selects a
+target — every such decision is already in the bundle.
+
+Build is not load: the installer runs generated create DDL, creates folder
+directories, deploys an item's runtime code, and reconciles the target — it never
+executes an object's code, and it has no route back to the source repository at
+all, because a bundle carries its outputs rather than a second copy of its
+inputs. Concurrency starts conservatively:
+sequences are serial and actions run serially within a batch, because one shared
+local Spark session gives no useful parallel DDL. The manifest still models
+independent actions, so a Fabric installer can add session concurrency later
+without changing bundle semantics.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Mapping
+
+from ..errors import InstallError
+from ..locations import Location
+from ..store import Store
+from ..targets import ItemRef
+from .bundle import BuildBundle, load_bundle, validate_bundle
+from .executors import default_executors
+from .executors.base import (
+    ActionExecutor,
+    InstallationContext,
+    ResolvedTarget,
+    SkippedExecution,
+)
+from .models import BuildAction, BuildBatch, BuildPlan, BuildSequence
+from .report import (
+    FAILED,
+    SKIPPED,
+    SUCCEEDED,
+    ActionResult,
+    InstallationReport,
+    SequenceResult,
+)
+from .targets import WAREHOUSE_TARGET, BoundTarget
+
+REPORT_FILENAME = "install-report.yml"
+
+
+@dataclass
+class InstallationEnvironment:
+    """Runtime services the installer executes against — no planning inputs.
+
+    ``spark`` is optional so a Folder-only bundle needs no session; a bundle
+    with Spark work supplies one. ``sql`` is likewise optional: a Warehouse
+    install acquires it **Fabric-natively** from the session identity, and only a
+    desktop caller crossing into Fabric injects ``desktop_sql_executor``
+    explicitly (``workspace`` is then unnecessary). ``executors`` defaults to the
+    built-in registry.
+    """
+
+    store: Store
+    resolver: Any
+    spark: Any = None
+    sql: Any = None
+    workspace: Any = None
+    executors: dict[str, ActionExecutor] = field(default_factory=default_executors)
+    #: Set when this environment opened its own Fabric-native SQL, so it closes it.
+    _owned_sql: Any = field(default=None, init=False, repr=False)
+
+    def resolve_target(self, bound: BoundTarget) -> ResolvedTarget:
+        # The resolver, store and Spark already define the environment the
+        # installer is running in, so a target is its item plus that item's
+        # physical roots. Resolving here, once, is what stops an executor deriving
+        # a path for itself — and what would let one installation address several
+        # destination Lakehouses without ever changing the session's own.
+        item = ItemRef(bound.item_id)
+        return ResolvedTarget(
+            bound=bound,
+            lakehouse=item,
+            location=self._resolved(bound, item, "lakehouse_spark_location"),
+            destination=self._resolved(bound, item, "spark_destination"),
+        )
+
+    def _resolved(self, bound: BoundTarget, item: ItemRef, method: str):
+        """One of the destination's two addresses, where the workspace can give it.
+
+        A Warehouse has neither — it is reached over TDS — and a resolver may not
+        implement the method at all. Neither is a failure here, because the
+        actions that need an address are Lakehouse actions and each fails
+        explicitly, naming the target, when it is missing.
+        """
+
+        if bound.kind == WAREHOUSE_TARGET:
+            return None
+        resolve = getattr(self.resolver, method, None)
+        if resolve is None:
+            return None
+        return resolve(item)
+
+    def sql_for(self, bound: BoundTarget) -> Any:
+        """The SQL capability for a Warehouse batch — injected, or Fabric-native.
+
+        Weaver runs in Fabric, so an install against a Warehouse authenticates
+        through the session's own identity rather than a desktop connection. The
+        executor is opened once per installation and closed with it.
+        """
+
+        if self.sql is not None:
+            return self.sql
+        if bound.kind != WAREHOUSE_TARGET:
+            return None
+        if self._owned_sql is None:
+            from ..fabric.sql import fabric_sql_executor
+            from ..targets import WarehouseTarget
+
+            self._owned_sql = fabric_sql_executor(
+                WarehouseTarget(warehouse=ItemRef(bound.item_id)), self.workspace
+            )
+        return self._owned_sql
+
+    def close(self) -> None:
+        if self._owned_sql is not None and hasattr(self._owned_sql, "close"):
+            self._owned_sql.close()
+            self._owned_sql = None
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _epoch(started: datetime) -> str:
+    """This installation's instant, as a Spark timestamp literal.
+
+    Naive rather than offset-carrying: the column is a plain ``timestamp`` and a
+    trailing offset would be parsed against the session's zone, which differs
+    between a desktop and a Fabric driver. UTC throughout, spelled without one.
+    """
+
+    return started.strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def install_bundle(
+    bundle: BuildBundle | Location,
+    *,
+    environment: InstallationEnvironment,
+) -> InstallationReport:
+    """Validate and run a bundle, returning a complete report."""
+
+    if isinstance(bundle, Location):
+        bundle = load_bundle(bundle, store=environment.store)
+    else:
+        # Preflight even a pre-loaded bundle: the installer trusts nothing it has
+        # not just checked.
+        validate_bundle(
+            bundle.location,
+            bundle.plan,
+            store=bundle.store or environment.store,
+        )
+
+    plan = bundle.plan
+    resolved = {target.id: environment.resolve_target(target) for target in plan.targets}
+
+    started = _now()
+    # One instant for the whole installation, taken once and handed to every
+    # batch. Registry rows are published across several statements — one pair per
+    # item — and rows written by one build have to be indistinguishable in age,
+    # or an alias and the source it points at could order against each other
+    # merely for having been written a few milliseconds apart.
+    epoch = _epoch(started)
+    sequence_results: list[SequenceResult] = []
+    stop = False
+
+    try:
+        for sequence in plan.sequences:
+            if stop:
+                sequence_results.append(_skipped_sequence(sequence))
+                continue
+            result = _run_sequence(sequence, resolved, bundle, environment, epoch=epoch)
+            sequence_results.append(result)
+            if result.status == FAILED:
+                stop = True
+    finally:
+        # Release any SQL connection this installation opened for itself.
+        environment.close()
+
+    finished = _now()
+    report = InstallationReport(
+        bundle_id=plan.bundle_id,
+        status=FAILED if stop else SUCCEEDED,
+        started_at=started,
+        finished_at=finished,
+        sequences=tuple(sequence_results),
+    )
+    (bundle.store or environment.store).write(
+        bundle.location.join(REPORT_FILENAME), report.to_yaml().encode("utf-8")
+    )
+    return report
+
+
+def _run_sequence(
+    sequence: BuildSequence,
+    resolved: dict[str, ResolvedTarget],
+    bundle: BuildBundle,
+    environment: InstallationEnvironment,
+    *,
+    epoch: str | None = None,
+) -> SequenceResult:
+    action_results: list[ActionResult] = []
+    failed = False
+
+    for batch in sequence.batches:
+        target = resolved[batch.target_id]
+        context = InstallationContext(
+            spark=environment.spark,
+            resolver=environment.resolver,
+            store=environment.store,
+            target=target,
+            sql=environment.sql_for(target.bound),
+            targets=resolved,
+            epoch=epoch,
+        )
+        for action in batch.actions:
+            if failed:
+                action_results.append(_skipped_action(action, batch))
+                continue
+            result = _run_action(action, batch, context, bundle, environment)
+            action_results.append(result)
+            if result.status == FAILED:
+                failed = True
+
+    skipped = bool(action_results) and all(
+        result.status == SKIPPED for result in action_results
+    )
+    return SequenceResult(
+        number=sequence.number,
+        description=sequence.description,
+        status=FAILED if failed else SKIPPED if skipped else SUCCEEDED,
+        actions=tuple(action_results),
+    )
+
+
+def execute_action(
+    action: BuildAction,
+    payload: bytes | None = None,
+    *,
+    context: InstallationContext,
+    executors: Mapping[str, ActionExecutor] | None = None,
+) -> ActionResult:
+    """Run one action against one target, with the installer's result semantics.
+
+    The same execution the installer performs, minus everything that is about a
+    *bundle*: no loading, no validation, no sequence barriers, no target
+    resolution, no report. One action, one payload, one context, one result.
+
+    It exists so the platform boundary can be tested where it actually is. A test
+    asking whether Fabric accepts Weaver's generated T-SQL, or whether a created
+    Warehouse table shows up in inventory, needs the statement *executed* — not a
+    repository parsed, a catalogue read, a bundle planned and an installation
+    reported. Those are separately proven, and running them again to reach the
+    one question costs a full build.
+
+    A failing action is data here exactly as it is in an installation: the error
+    is recorded on the result rather than raised, so the caller asserts on a
+    result in both places and the semantics cannot drift apart.
+    """
+
+    return _execute(
+        action,
+        lambda: payload,
+        context=context,
+        target_id=context.target.bound.id,
+        executors=default_executors() if executors is None else executors,
+    )
+
+
+def _run_action(
+    action: BuildAction,
+    batch: BuildBatch,
+    context: InstallationContext,
+    bundle: BuildBundle,
+    environment: InstallationEnvironment,
+) -> ActionResult:
+    def load_payload() -> bytes | None:
+        if action.payload is None:
+            return None
+        return (bundle.store or environment.store).read(
+            bundle.location.join(*action.payload.split("/"))
+        )
+
+    return _execute(
+        action,
+        load_payload,
+        context=context,
+        target_id=batch.target_id,
+        executors=environment.executors,
+    )
+
+
+def _execute(
+    action: BuildAction,
+    load_payload,
+    *,
+    context: InstallationContext,
+    target_id: str,
+    executors: Mapping[str, ActionExecutor],
+) -> ActionResult:
+    """The one place an action is run, shared by the installer and by callers.
+
+    ``load_payload`` is deferred rather than passed as bytes because a payload
+    that cannot be read is an action failure like any other, recorded with the
+    same timing and the same result shape as one whose executor raised.
+    """
+
+    started = _now()
+    executor = executors.get(action.executor)
+    if executor is None:
+        return _failed(
+            action, target_id, started, InstallError(f"no executor named {action.executor!r}")
+        )
+
+    try:
+        execution = executor.execute(action, load_payload(), context)
+    except Exception as exc:  # a failing action is data, not a crash
+        return _failed(action, target_id, started, exc)
+
+    finished = _now()
+    skipped = isinstance(execution, SkippedExecution)
+    return ActionResult(
+        action_id=action.id,
+        resource_node_id=action.resource_node_id,
+        target_id=target_id,
+        executor=action.executor,
+        status=SKIPPED if skipped else SUCCEEDED,
+        started_at=started,
+        finished_at=finished,
+        duration_seconds=(finished - started).total_seconds(),
+        details=execution.details if skipped else (execution or None),
+    )
+
+
+def _failed(
+    action: BuildAction, target_id: str, started: datetime, exc: Exception
+) -> ActionResult:
+    finished = _now()
+    return ActionResult(
+        action_id=action.id,
+        resource_node_id=action.resource_node_id,
+        target_id=target_id,
+        executor=action.executor,
+        status=FAILED,
+        started_at=started,
+        finished_at=finished,
+        duration_seconds=(finished - started).total_seconds(),
+        error_type=type(exc).__name__,
+        error_message=str(exc),
+    )
+
+
+def _skipped_action(action: BuildAction, batch: BuildBatch) -> ActionResult:
+    return ActionResult(
+        action_id=action.id,
+        resource_node_id=action.resource_node_id,
+        target_id=batch.target_id,
+        executor=action.executor,
+        status=SKIPPED,
+    )
+
+
+def _skipped_sequence(sequence: BuildSequence) -> SequenceResult:
+    actions = tuple(
+        _skipped_action(action, batch)
+        for batch in sequence.batches
+        for action in batch.actions
+    )
+    return SequenceResult(
+        number=sequence.number,
+        description=sequence.description,
+        status=SKIPPED,
+        actions=actions,
+    )

weaver/build_bundle/models.py

@@ -0,0 +1,288 @@
+"""The build manifest — immutable plan, sequence, batch and action types.
+
+A :class:`BuildPlan` is the whole deployment, fully bound: every executable
+identifies one physical target and carries everything an installer needs. It is
+produced once, by the planner, and thereafter only read.
+
+The execution shape is deliberately flat and explicit:
+
+- **sequences are barriers.** They run in order; the next begins only when every
+  action in the current one has succeeded.
+- **batches are target-bound.** A batch names exactly one target, so a physical
+  destination appears once per batch rather than being repeated inside actions.
+- **actions are independent units.** Each has its own payload (where one is
+  required), and is reported on its own.
+
+Every type serialises to and from plain mappings, which is what the canonical
+``plan.yml`` and the ``bundle_id`` hash are built from — see :mod:`weaver.build_bundle.bundle`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Mapping
+
+from .targets import BoundTarget
+from .changes import TargetChange
+from .incremental import BuildSelection
+
+#: Action kinds. Create kinds build structure; prune kinds reconcile the target.
+CREATE_SCHEMA = "create_schema"
+CREATE_ALIAS = "create_alias"
+BUILD_FOLDER = "build_folder"
+BUILD_TABLE = "build_table"
+BUILD_VIEW = "build_view"
+
+#: One Lakehouse's SQL analytics endpoint catching up with the Delta mutations
+#: just made in it. It closes an item's physical work: a dependent item's
+#: Warehouse view or OneLake shortcut reads that endpoint's metadata, so it must
+#: not be created while the endpoint still describes the previous shape.
+REFRESH_SQL_ENDPOINT = "refresh_sql_endpoint"
+
+#: Load kinds — what an item's final layer installs. ``write_file`` puts one
+#: deployed module or generated statement into the runtime tree;
+#: ``build_procedure`` creates or replaces one generated load procedure.
+WRITE_FILE = "write_file"
+BUILD_PROCEDURE = "build_procedure"
+
+#: Managed rebuild drops.  They are deliberately distinct from prune: these
+#: objects remain desired and are removed only so a selected definition can be
+#: recreated.
+DROP_FOLDER = "drop_folder"
+DROP_TABLE = "drop_table"
+DROP_VIEW = "drop_view"
+
+#: Removals of load artefacts whose source has stopped claiming them. Distinct
+#: from prune because they come from the *catalogue* rather than from a diff
+#: against the target: the previous Registry row says what was installed and
+#: where, so a deleted or renamed source produces the removal without anything
+#: having to enumerate the runtime tree.
+DELETE_FILE = "delete_file"
+DROP_PROCEDURE = "drop_procedure"
+
+#: Prune kinds. Each names one frozen drop the build computed against the target:
+#: a Spark SQL DROP for a table/view/schema, a directory removal for a folder.
+PRUNE_TABLE = "prune_table"
+PRUNE_VIEW = "prune_view"
+PRUNE_SCHEMA = "prune_schema"
+PRUNE_FOLDER = "prune_folder"
+
+#: Refresh the SQL analytics endpoint for one Lakehouse after its Delta tables
+#: have changed. The action is target-bound and payloadless: the planner decides
+#: which Lakehouses need it, while the executor performs or explicitly skips it
+#: for the current host.
+REFRESH_SQL_ENDPOINT = "refresh_sql_endpoint"
+
+#: Catalogue kinds. These write the central catalogue in the Weaver Lakehouse
+#: rather than the destination. Claim deletion leads physical work; batched
+#: publication concludes it, with Registry visibly last in the manifest.
+DELETE_CATALOGUE_CLAIMS = "delete_catalogue_claims"
+PUBLISH_CATALOGUE = "publish_catalogue"
+PUBLISH_REGISTRY = "publish_registry"
+CATALOGUE_KINDS = frozenset(
+    {
+        DELETE_CATALOGUE_CLAIMS,
+        PUBLISH_CATALOGUE,
+        PUBLISH_REGISTRY,
+    }
+)
+
+#: Reasons a repository node is not in the plan. A missing target is visible,
+#: not a mysterious absence.
+OMIT_TARGET_UNBOUND = "target_unbound"
+OMIT_DEPENDS_ON_OMITTED = "depends_on_omitted_node"
+OMIT_UNSUPPORTED_EXECUTOR = "unsupported_executor"
+#: An alias the current bindings give no physical form. The planner decides this
+#: — never the installer, which may only run an alias action already frozen for
+#: it — and records it so the absence is a stated decision rather than a gap.
+OMIT_ALIAS_UNSUPPORTED = "alias_unsupported"
+OMISSION_REASONS = frozenset(
+    {
+        OMIT_TARGET_UNBOUND,
+        OMIT_DEPENDS_ON_OMITTED,
+        OMIT_UNSUPPORTED_EXECUTOR,
+        OMIT_ALIAS_UNSUPPORTED,
+    }
+)
+
+
+@dataclass(frozen=True)
+class OmittedNode:
+    """A repository node the projection left out, and why."""
+
+    node_id: str
+    reason: str
+    detail: str | None = None
+
+    def to_mapping(self) -> dict[str, Any]:
+        mapping: dict[str, Any] = {"node_id": self.node_id, "reason": self.reason}
+        if self.detail is not None:
+            mapping["detail"] = self.detail
+        return mapping
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "OmittedNode":
+        return cls(
+            node_id=mapping["node_id"],
+            reason=mapping["reason"],
+            detail=mapping.get("detail"),
+        )
+
+
+@dataclass(frozen=True)
+class BuildAction:
+    """One independently executable unit.
+
+    ``payload`` is a bundle-relative path to the generated definition, or None
+    for an action that carries no payload (an explicit no-op). ``payload_sha256``
+    hashes that payload so corruption is caught before anything runs.
+    """
+
+    id: str
+    kind: str
+    resource_node_id: str | None
+    executor: str
+    payload: str | None
+    payload_sha256: str | None
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "kind": self.kind,
+            "resource_node_id": self.resource_node_id,
+            "executor": self.executor,
+            "payload": self.payload,
+            "payload_sha256": self.payload_sha256,
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "BuildAction":
+        return cls(
+            id=mapping["id"],
+            kind=mapping["kind"],
+            resource_node_id=mapping.get("resource_node_id"),
+            executor=mapping["executor"],
+            payload=mapping.get("payload"),
+            payload_sha256=mapping.get("payload_sha256"),
+        )
+
+
+@dataclass(frozen=True)
+class BuildBatch:
+    """A group of actions against exactly one target."""
+
+    id: str
+    target_id: str
+    actions: tuple[BuildAction, ...]
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "target_id": self.target_id,
+            "actions": [action.to_mapping() for action in self.actions],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "BuildBatch":
+        return cls(
+            id=mapping["id"],
+            target_id=mapping["target_id"],
+            actions=tuple(BuildAction.from_mapping(a) for a in mapping.get("actions", ())),
+        )
+
+
+@dataclass(frozen=True)
+class BuildSequence:
+    """One barrier. Every batch here completes before the next sequence starts."""
+
+    number: int
+    description: str
+    batches: tuple[BuildBatch, ...]
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "number": self.number,
+            "description": self.description,
+            "batches": [batch.to_mapping() for batch in self.batches],
+        }
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "BuildSequence":
+        return cls(
+            number=mapping["number"],
+            description=mapping["description"],
+            batches=tuple(BuildBatch.from_mapping(b) for b in mapping.get("batches", ())),
+        )
+
+
+@dataclass(frozen=True)
+class BuildPlan:
+    """A whole deployment, fully bound and ordered."""
+
+    format_version: int
+    bundle_id: str
+    repository_name: str
+    repository_signature: str
+    targets: tuple[BoundTarget, ...]
+    sequences: tuple[BuildSequence, ...]
+    selection: BuildSelection
+    omitted_nodes: tuple[OmittedNode, ...] = ()
+    #: What this plan will *mean* for each bound target, keyed by target id —
+    #: the objects it adds and removes. Part of the manifest, and therefore of
+    #: the bundle identity, so the summary a reviewer reads is the summary the
+    #: installation was certified with. A sibling file outside the hash could be
+    #: edited after certification, which is the thing frozen payloads exist to
+    #: prevent.
+    target_changes: Mapping[str, tuple[TargetChange, ...]] = field(
+        default_factory=dict
+    )
+
+    def to_mapping(self) -> dict[str, Any]:
+        mapping = {
+            "format_version": self.format_version,
+            "bundle_id": self.bundle_id,
+            "repository_name": self.repository_name,
+            "repository_signature": self.repository_signature,
+            "targets": [target.to_mapping() for target in self.targets],
+            "sequences": [sequence.to_mapping() for sequence in self.sequences],
+            "omitted_nodes": [node.to_mapping() for node in self.omitted_nodes],
+            "target_changes": {
+                target_id: [change.to_mapping() for change in changes]
+                for target_id, changes in sorted(self.target_changes.items())
+            },
+        }
+        mapping["selection"] = self.selection.to_mapping()
+        return mapping
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "BuildPlan":
+        return cls(
+            format_version=mapping["format_version"],
+            bundle_id=mapping["bundle_id"],
+            repository_name=mapping["repository_name"],
+            repository_signature=mapping["repository_signature"],
+            targets=tuple(BoundTarget.from_mapping(t) for t in mapping.get("targets", ())),
+            sequences=tuple(BuildSequence.from_mapping(s) for s in mapping.get("sequences", ())),
+            selection=BuildSelection.from_mapping(mapping["selection"]),
+            omitted_nodes=tuple(
+                OmittedNode.from_mapping(n) for n in mapping.get("omitted_nodes", ())
+            ),
+            target_changes={
+                target_id: tuple(TargetChange.from_mapping(c) for c in changes)
+                for target_id, changes in mapping.get("target_changes", {}).items()
+            },
+        )
+
+    # --- convenience views ------------------------------------------------
+
+    @property
+    def target_ids(self) -> frozenset[str]:
+        return frozenset(target.id for target in self.targets)
+
+    def actions(self):
+        """Every action, in manifest order."""
+
+        for sequence in self.sequences:
+            for batch in sequence.batches:
+                for action in batch.actions:
+                    yield sequence, batch, action

weaver/build_bundle/payloads.py

@@ -0,0 +1,34 @@
+"""Payload naming and hashing — where a generated definition lives in a bundle.
+
+A bundle's ``payload/`` tree groups definitions by the sequence that runs them,
+so the directory order mirrors the deployment order and a reviewer can read it
+top to bottom. This module owns those names and the payload hash, so the planner
+does not scatter path arithmetic through its logic.
+
+There are deliberately no reserved sequence numbers here. A stage's number comes
+from its position in the assembled plan — see :mod:`weaver.build_bundle.stages`
+— so nothing has to leave arithmetic headroom for a repository's dependency depth
+and nothing can collide with a region another phase claimed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+PAYLOAD_ROOT = "payload"
+
+
+def sequence_dir(number: int, slug: str) -> str:
+    """The payload subdirectory for one sequence, e.g. ``003-build-objects``."""
+
+    return f"{number:03d}-{slug}"
+
+
+def payload_path(number: int, slug: str, filename: str) -> str:
+    """A bundle-relative payload path under its sequence directory."""
+
+    return f"{PAYLOAD_ROOT}/{sequence_dir(number, slug)}/{filename}"
+
+
+def sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()