weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/report.py

@@ -0,0 +1,108 @@
+"""The installation report — one result per action, faithfully.
+
+An install is judged by its report, so the report must be exact: every planned
+action gets exactly one result, with its status, timing and — on failure — the
+error, and a sequence that never started is recorded as skipped rather than
+omitted. The whole thing serialises so a local run can drop an
+``install-report.yml`` beside the plan; on Fabric the same structure can move to
+control tables later.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+PENDING = "pending"
+RUNNING = "running"
+SUCCEEDED = "succeeded"
+FAILED = "failed"
+SKIPPED = "skipped"
+
+
+def _iso(value: datetime | None) -> str | None:
+    return value.isoformat() if value is not None else None
+
+
+@dataclass(frozen=True)
+class ActionResult:
+    """The outcome of one action."""
+
+    action_id: str
+    resource_node_id: str | None
+    target_id: str
+    executor: str
+    status: str
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    duration_seconds: float | None = None
+    error_type: str | None = None
+    error_message: str | None = None
+    details: dict[str, Any] | None = None
+
+    def to_mapping(self) -> dict[str, Any]:
+        mapping: dict[str, Any] = {
+            "action_id": self.action_id,
+            "resource_node_id": self.resource_node_id,
+            "target_id": self.target_id,
+            "executor": self.executor,
+            "status": self.status,
+            "started_at": _iso(self.started_at),
+            "finished_at": _iso(self.finished_at),
+            "duration_seconds": self.duration_seconds,
+        }
+        if self.error_type is not None:
+            mapping["error_type"] = self.error_type
+            mapping["error_message"] = self.error_message
+        if self.details:
+            mapping["details"] = self.details
+        return mapping
+
+
+@dataclass(frozen=True)
+class SequenceResult:
+    number: int
+    description: str
+    status: str
+    actions: tuple[ActionResult, ...]
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "number": self.number,
+            "description": self.description,
+            "status": self.status,
+            "actions": [action.to_mapping() for action in self.actions],
+        }
+
+
+@dataclass(frozen=True)
+class InstallationReport:
+    bundle_id: str
+    status: str
+    started_at: datetime
+    finished_at: datetime | None
+    sequences: tuple[SequenceResult, ...]
+
+    @property
+    def succeeded(self) -> bool:
+        return self.status == SUCCEEDED
+
+    def action_results(self):
+        for sequence in self.sequences:
+            for action in sequence.actions:
+                yield action
+
+    def to_mapping(self) -> dict[str, Any]:
+        return {
+            "bundle_id": self.bundle_id,
+            "status": self.status,
+            "started_at": _iso(self.started_at),
+            "finished_at": _iso(self.finished_at),
+            "sequences": [sequence.to_mapping() for sequence in self.sequences],
+        }
+
+    def to_yaml(self) -> str:
+        import yaml
+
+        return yaml.safe_dump(self.to_mapping(), sort_keys=False, allow_unicode=True)

weaver/build_bundle/stages.py

@@ -0,0 +1,196 @@
+"""Logical stages, and the one place a sequence number is chosen.
+
+A planning component answers *what* has to happen and in what order relative to
+its own siblings. It does not answer *which sequence number* that is, because a
+number is a property of the finished plan and nothing else: with one alias, one
+schema and one endpoint-refresh stage per item, arithmetic over reserved regions
+stops describing the plan and starts constraining it.
+
+So each component returns :class:`PlannedStage` values — a phase, a description,
+target-bound batches, and the payloads those batches need, keyed by bare
+filename. The top-level planner concatenates the stages in execution order and
+:func:`enumerate_stages` turns them into :class:`~weaver.build_bundle.models.BuildSequence`
+values, numbering them 1, 2, 3 … and rewriting each payload into
+``payload/<number>-<slug>/<filename>`` so the bundle directory still reads top to
+bottom in deployment order.
+
+Numbering last also means a stage cannot collide with another stage's region, and
+there is no headroom to run out of: the number *describes* the order the plan
+already has.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import Iterable, Mapping, Sequence
+
+from ..errors import BuildError
+from .changes import TargetChange, merge as merge_changes
+from .models import BuildBatch, BuildSequence
+from .payloads import payload_path
+
+#: The phases one item's work is made of, in the order they must run.
+#:
+#: Prune and managed drops come first because they are the destructive
+#: reconciliation of what is already there. Schemas precede aliases so an alias
+#: materialised as a Warehouse view has a schema to be created in, and aliases
+#: precede builds so every document this item declares is built against a
+#: namespace that already holds what the item imports. The refresh closes the
+#: item: until a mutated Lakehouse's SQL endpoint has caught up, a dependent
+#: item's view or shortcut would be built over metadata that does not describe it.
+#: Load closes the item, after the refresh. Its artefacts depend on the item's
+#: structural work being finished and on nothing within their own layer — a
+#: deployed module and a generated procedure have no ordering between them,
+#: because nothing here runs them.
+PRUNE = "prune"
+DROP = "drop"
+SCHEMA = "schema"
+ALIAS = "alias"
+BUILD = "build"
+REFRESH = "refresh"
+LOAD = "load"
+CATALOGUE = "catalogue"
+
+_PHASE_ORDER = (PRUNE, DROP, SCHEMA, ALIAS, BUILD, REFRESH, LOAD, CATALOGUE)
+_PHASE_RANK = {phase: rank for rank, phase in enumerate(_PHASE_ORDER)}
+
+
+@dataclass(frozen=True)
+class PlannedStage:
+    """One barrier's worth of work, before it is given a number.
+
+    ``phase`` and ``index`` place the stage among its siblings: ``index``
+    separates the dependency layers within a phase, so two items in the same
+    topological item layer can have their layer *n* merged into one barrier.
+
+    ``slug`` names the stage's payload directory. ``payloads`` is keyed by bare
+    filename within it, because the directory's name is not known until the
+    stage has a number.
+
+    ``changes`` is what this stage's actions will *mean* for each target, keyed
+    by target id. Rendered beside the actions rather than inferred from them —
+    see :mod:`weaver.build_bundle.changes` — so the statement of effect and the
+    thing that has the effect are written in one place.
+    """
+
+    phase: str
+    description: str
+    batches: tuple[BuildBatch, ...]
+    slug: str = ""
+    index: int = 0
+    payloads: Mapping[str, bytes] = field(default_factory=dict)
+    changes: Mapping[str, tuple[TargetChange, ...]] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if self.phase not in _PHASE_RANK:
+            raise BuildError(f"unknown planned stage phase {self.phase!r}")
+        for filename in self.payloads:
+            if "/" in filename or not filename:
+                raise BuildError(
+                    f"stage {self.phase!r} payload key must be a bare filename, "
+                    f"got {filename!r}"
+                )
+
+    @property
+    def payload_slug(self) -> str:
+        return self.slug or self.phase
+
+    @property
+    def rank(self) -> tuple[int, int]:
+        return (_PHASE_RANK[self.phase], self.index)
+
+
+def merge_layer_stages(stages: Iterable[PlannedStage]) -> tuple[PlannedStage, ...]:
+    """Fold same-phase, same-index stages from one item layer into one barrier.
+
+    Items in the same topological layer have no ordering between them, so their
+    work belongs in the same barriers: one batch per item, exactly as a
+    single-layer build already produces. Merging here is what keeps the
+    invariant that matters — nothing in a later item layer starts before this
+    layer has completed — without serialising items that never needed it.
+    """
+
+    grouped: dict[tuple[int, int], list[PlannedStage]] = {}
+    for stage in stages:
+        grouped.setdefault(stage.rank, []).append(stage)
+
+    merged: list[PlannedStage] = []
+    for rank in sorted(grouped):
+        group = grouped[rank]
+        first = group[0]
+        payloads: dict[str, bytes] = {}
+        for stage in group:
+            if stage.payload_slug != first.payload_slug:
+                raise BuildError(
+                    f"stages merged into one barrier disagree about their payload "
+                    f"directory: {first.payload_slug!r} and {stage.payload_slug!r}"
+                )
+            for filename, content in stage.payloads.items():
+                if payloads.setdefault(filename, content) != content:
+                    raise BuildError(
+                        f"two merged stages disagree about payload {filename!r}"
+                    )
+        merged.append(
+            replace(
+                first,
+                batches=tuple(batch for stage in group for batch in stage.batches),
+                payloads=payloads,
+                changes=merge_changes(*(stage.changes for stage in group)),
+            )
+        )
+    return tuple(merged)
+
+
+def enumerate_stages(
+    stages: Sequence[PlannedStage],
+) -> tuple[
+    tuple[BuildSequence, ...],
+    dict[str, bytes],
+    dict[str, tuple[TargetChange, ...]],
+]:
+    """Number the assembled plan and resolve every payload path.
+
+    Batch ids gain the same number prefix, so a batch is still identifiable in a
+    report and still unique across the plan without any component having to know
+    what else is being planned.
+    """
+
+    sequences: list[BuildSequence] = []
+    payloads: dict[str, bytes] = {}
+    changes: list[Mapping[str, tuple[TargetChange, ...]]] = []
+    # An empty stage is not a barrier — it is a phase this build had no work for
+    # — so it takes no number and leaves no gap.
+    populated = [stage for stage in stages if stage.batches]
+    for number, stage in enumerate(populated, start=1):
+        resolved = {}
+        for filename, content in stage.payloads.items():
+            path = payload_path(number, stage.payload_slug, filename)
+            resolved[filename] = path
+            payloads[path] = content
+        changes.append(stage.changes)
+        sequences.append(
+            BuildSequence(
+                number=number,
+                description=stage.description,
+                batches=tuple(
+                    _numbered(batch, number, resolved) for batch in stage.batches
+                ),
+            )
+        )
+    return tuple(sequences), payloads, merge_changes(*changes)
+
+
+def _numbered(batch: BuildBatch, number: int, payloads: Mapping[str, str]) -> BuildBatch:
+    actions = []
+    for action in batch.actions:
+        if action.payload is None:
+            actions.append(action)
+            continue
+        resolved = payloads.get(action.payload)
+        if resolved is None:
+            raise BuildError(
+                f"action {action.id!r} names payload {action.payload!r}, which its "
+                "stage did not supply"
+            )
+        actions.append(replace(action, payload=resolved))
+    return replace(batch, id=f"{number:03d}-{batch.id}", actions=tuple(actions))

weaver/build_bundle/targets.py

@@ -0,0 +1,272 @@
+"""Serialisable physical target descriptors.
+
+A build request supplies live workspace objects; a bundle must not. The planner
+converts each supplied binding into a :class:`BoundTarget` — a flat, stable
+descriptor carrying exactly what an installer needs to resolve the physical
+destination, and nothing that ties the bundle to the process that wrote it.
+
+There is no workspace kind here. Weaver has one real workspace, Fabric; local execution is
+an emulation of it for development, not a second kind the bundle contract records.
+A target names an item — a Lakehouse or a Warehouse — by the identifiers the
+installer resolves it with; where the installer is running (in a Fabric session,
+or in-process locally) is supplied by its environment, not frozen into the bundle.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping
+
+from ..targets import ItemRef
+from ..errors import BuildError
+from ..declaration.model import LAKEHOUSE, WAREHOUSE, WeaverItemId
+
+#: Target kinds a bound target may name. They mirror the Weaver document target kinds but
+#: live here because a bundle is read without importing the Weaver document vocabulary.
+LAKEHOUSE_TARGET = "lakehouse"
+WAREHOUSE_TARGET = "warehouse"
+
+
+@dataclass(frozen=True)
+class BoundTarget:
+    """One physical destination, as flat serialisable data.
+
+    ``id`` is the manifest-local identifier a batch names. ``kind`` says whether
+    it is a Lakehouse or a Warehouse. ``item_id`` names the item, with the
+    optional Fabric identifiers alongside; the installer resolves the item
+    through its own environment.
+    """
+
+    id: str
+    kind: str
+    item_id: str
+    #: The item's resolved display name. Carried alongside ``item_id`` because on
+    #: Fabric the id is a GUID: the catalogue records which item an installation is
+    #: bound to, and a GUID would make that record unreadable. It is a *record*,
+    #: never identity — resolution goes through ``item_id``.
+    item_name: str | None = None
+    workspace_id: str | None = None
+    sql_endpoint_id: str | None = None
+    logical_item_type: str | None = None
+    logical_item_name: str | None = None
+
+    @property
+    def name(self) -> str:
+        """The readable name, falling back to the id when none was carried."""
+
+        return self.item_name or self.item_id
+
+    def to_mapping(self) -> dict[str, Any]:
+        mapping: dict[str, Any] = {
+            "id": self.id,
+            "kind": self.kind,
+            "item_id": self.item_id,
+        }
+        if self.item_name is not None:
+            mapping["item_name"] = self.item_name
+        if self.workspace_id is not None:
+            mapping["workspace_id"] = self.workspace_id
+        if self.sql_endpoint_id is not None:
+            mapping["sql_endpoint_id"] = self.sql_endpoint_id
+        if self.logical_item_type is not None:
+            mapping["logical_item_type"] = self.logical_item_type
+        if self.logical_item_name is not None:
+            mapping["logical_item_name"] = self.logical_item_name
+        return mapping
+
+    @classmethod
+    def from_mapping(cls, mapping: Mapping[str, Any]) -> "BoundTarget":
+        return cls(
+            id=mapping["id"],
+            kind=mapping["kind"],
+            item_id=mapping["item_id"],
+            item_name=mapping.get("item_name"),
+            workspace_id=mapping.get("workspace_id"),
+            sql_endpoint_id=mapping.get("sql_endpoint_id"),
+            logical_item_type=mapping.get("logical_item_type"),
+            logical_item_name=mapping.get("logical_item_name"),
+        )
+
+
+# --- input bindings ----------------------------------------------------------
+#
+# What a caller supplies to the planner. These carry a live identity (an
+# ItemRef, and for Fabric the workspace/item ids); the planner converts them into
+# the flat BoundTarget above so no live workspace object is serialised into a bundle.
+
+
+@dataclass(frozen=True)
+class LakehouseBinding:
+    """A bound destination Lakehouse for Folder and Delta materialisation."""
+
+    lakehouse: ItemRef
+    workspace_id: str | None = None
+    #: The concrete Fabric item id; locally the logical Lakehouse name serves.
+    item_id: str | None = None
+
+    def to_bound_target(self) -> BoundTarget:
+        return BoundTarget(
+            id=f"{LAKEHOUSE_TARGET}-{self.lakehouse.name}",
+            kind=LAKEHOUSE_TARGET,
+            item_id=self.item_id or self.lakehouse.name,
+            item_name=self.lakehouse.name,
+            workspace_id=self.workspace_id,
+        )
+
+
+@dataclass(frozen=True)
+class WarehouseBinding:
+    """A bound destination Warehouse. Present so the boundary is visible; v1
+    installation of Warehouse work is not supported and raises."""
+
+    warehouse: ItemRef
+    workspace_id: str | None = None
+    item_id: str | None = None
+    sql_endpoint_id: str | None = None
+
+    def to_bound_target(self) -> BoundTarget:
+        return BoundTarget(
+            id=f"{WAREHOUSE_TARGET}-{self.warehouse.name}",
+            kind=WAREHOUSE_TARGET,
+            item_id=self.item_id or self.warehouse.name,
+            item_name=self.warehouse.name,
+            workspace_id=self.workspace_id,
+            sql_endpoint_id=self.sql_endpoint_id,
+        )
+
+
+@dataclass(frozen=True)
+class ItemBinding:
+    """One exact logical Weaver item bound to one typed physical item."""
+
+    item: WeaverItemId
+    target: LakehouseBinding | WarehouseBinding
+
+    def __post_init__(self) -> None:
+        expected = LAKEHOUSE if isinstance(self.target, LakehouseBinding) else WAREHOUSE
+        if self.item.item_type != expected:
+            raise BuildError(
+                f"logical item {self.item} requires a {self.item.item_type} binding, "
+                f"not {type(self.target).__name__}"
+            )
+
+    def to_bound_target(self) -> BoundTarget:
+        physical = self.target.to_bound_target()
+        logical_slug = f"{self.item.item_type}-{self.item.item_name}"
+        return BoundTarget(
+            id=f"{logical_slug}--{physical.id}",
+            kind=physical.kind,
+            item_id=physical.item_id,
+            item_name=physical.item_name,
+            workspace_id=physical.workspace_id,
+            sql_endpoint_id=physical.sql_endpoint_id,
+            logical_item_type=self.item.item_type,
+            logical_item_name=self.item.item_name,
+        )
+
+
+@dataclass(frozen=True)
+class ItemBindings:
+    """The sparse logical-to-physical bindings for one coordinated build."""
+
+    entries: tuple[ItemBinding, ...]
+
+    def __post_init__(self) -> None:
+        seen: set[WeaverItemId] = set()
+        physical: set[tuple[str, str]] = set()
+        for binding in self.entries:
+            if binding.item in seen:
+                raise BuildError(f"logical item is bound more than once: {binding.item}")
+            seen.add(binding.item)
+            target = binding.target
+            key = (
+                LAKEHOUSE if isinstance(target, LakehouseBinding) else WAREHOUSE,
+                target.lakehouse.name
+                if isinstance(target, LakehouseBinding)
+                else target.warehouse.name,
+            )
+            if key in physical:
+                raise BuildError(
+                    f"physical {key[0]} target is bound more than once: {key[1]}"
+                )
+            physical.add(key)
+
+    @property
+    def by_item(self) -> Mapping[WeaverItemId, ItemBinding]:
+        return {binding.item: binding for binding in self.entries}
+
+
+def effective_item_bindings(
+    bindings: ItemBindings, *, weaver_lakehouse: str
+) -> ItemBindings:
+    """Add the mandatory package-owned control item binding."""
+
+    builtin = WeaverItemId(LAKEHOUSE, "_weaver")
+    if builtin in bindings.by_item:
+        raise BuildError("Lakehouse/_weaver is bound implicitly and must not be selected")
+    return ItemBindings(
+        bindings.entries
+        + (
+            ItemBinding(
+                builtin,
+                LakehouseBinding(ItemRef.parse(weaver_lakehouse)),
+            ),
+        )
+    )
+
+
+def parse_item_binding(text: str, *, workspace=None) -> ItemBinding:
+    """Parse a typed physical selector with an optional logical override.
+
+    ``Lakehouse/Sales`` uses the configured default.  The self-contained form
+    ``Lakehouse/Sales=Lakehouse/Raw`` needs no configured target declaration.
+    """
+
+    if not isinstance(text, str) or text.count("=") > 1:
+        raise BuildError(
+            "a binding must be TypedPhysical/Name or TypedPhysical/Name=Logical/Item"
+        )
+    physical_text, separator, logical_text = text.partition("=")
+    physical_text = physical_text.strip()
+    logical_text = logical_text.strip()
+    if not physical_text or (separator and not logical_text):
+        raise BuildError(
+            "a binding must be TypedPhysical/Name or TypedPhysical/Name=Logical/Item"
+        )
+
+    physical_type, physical = _parse_physical_item(physical_text)
+    if separator:
+        item = WeaverItemId.parse(logical_text)
+    else:
+        if workspace is None:
+            raise BuildError(
+                f"binding {physical_text!r} needs a Workspace configuration default "
+                "or an explicit =Logical/Item"
+            )
+        item = workspace.declaration_for(physical_type, physical.name).item
+    if item.item_type != physical_type:
+        raise BuildError(
+            f"physical {physical_text} cannot be bound to logical {item}; "
+            f"both must be {physical_type}"
+        )
+    target = (
+        LakehouseBinding(physical)
+        if physical_type == LAKEHOUSE
+        else WarehouseBinding(physical)
+    )
+    return ItemBinding(item, target)
+
+
+def _parse_physical_item(text: str) -> tuple[str, ItemRef]:
+    """The binding's physical half, through the grammar every operation shares.
+
+    The logical item types and the grammar's spellings happen to be the same two
+    words, so the kind is used directly rather than translated.
+    """
+
+    from ..targets import parse_physical_target, physical_item, physical_kind
+
+    target = parse_physical_target(
+        text, what="binding physical target", error=BuildError
+    )
+    return physical_kind(target), physical_item(target)