weaverstack 0.1.1__py3-none-any.whl

weaver/build_bundle/bundle.py

@@ -0,0 +1,359 @@
+"""Writing, loading and validating a build bundle on a store.
+
+A bundle is a directory:
+
+.. code-block:: text
+
+    <bundle>/
+        plan.yml                     the canonical manifest
+        payload/                     one generated definition per action
+            010-create-schemas/
+                create-DWG.spark.sql
+            ...
+
+The manifest is written **last**, so a half-written directory never looks
+installable. Loading validates the whole bundle — structure, target bindings,
+payload presence, and payload hashes — before any action can run, because the
+installer must be able to trust what it is handed without re-reading the source.
+
+``bundle_id`` is derived from stable inputs only: the format version, the
+repository signature, the target descriptors and the canonical manifest with
+its payload hashes. No timestamp participates, so the same inputs always yield
+the same identity.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Mapping
+
+import yaml
+
+from ..errors import BuildError
+from ..locations import Location
+from ..store import Store
+from .models import DELETE_FILE, OMISSION_REASONS, BuildPlan
+
+#: The only bundle format this code writes and accepts.
+SUPPORTED_FORMAT_VERSION = 1
+
+PLAN_FILENAME = "plan.yml"
+PAYLOAD_DIR = "payload"
+
+SPARK_SQL_EXECUTOR = "spark_sql"
+SPARK_SQL_BATCH_EXECUTOR = "spark_sql_batch"
+SPARK_SCHEMA_EXECUTOR = "spark_schema"
+SPARK_TABLE_EXECUTOR = "spark_table"
+TSQL_EXECUTOR = "tsql"
+TSQL_BATCH_EXECUTOR = "tsql_batch"
+FOLDER_EXECUTOR = "folder"
+ALIAS_EXECUTOR = "alias"
+SQL_ENDPOINT_REFRESH_EXECUTOR = "sql_endpoint_refresh"
+LOAD_FILE_EXECUTOR = "load_file"
+#: Executors a bundle may carry. ``spark_sql`` runs a create or a frozen prune
+#: DROP; ``spark_sql_batch`` runs ordered catalogue DML as one action;
+#: ``spark_schema`` makes one schema in the destination, whose ``LOCATION``
+#: is a resolved path and so cannot be frozen; ``spark_table`` completes a Spark
+#: SQL table's deferred build by running its query and creating the table;
+#: ``tsql`` runs a self-contained Warehouse script and ``tsql_batch`` an
+#: ordered array of them, each as its own batch; ``folder`` makes or removes a
+#: directory; ``alias`` points one Lakehouse name at another item's object
+VALID_EXECUTORS = frozenset(
+    {
+        SPARK_SQL_EXECUTOR,
+        SPARK_SQL_BATCH_EXECUTOR,
+        SPARK_SCHEMA_EXECUTOR,
+        SPARK_TABLE_EXECUTOR,
+        TSQL_EXECUTOR,
+        TSQL_BATCH_EXECUTOR,
+        FOLDER_EXECUTOR,
+        ALIAS_EXECUTOR,
+        SQL_ENDPOINT_REFRESH_EXECUTOR,
+        LOAD_FILE_EXECUTOR,
+    }
+)
+#: Executors that run a payload, and the extension that payload must carry.
+#: ``folder`` acts on the resolved target and carries none; ``sql_endpoint_refresh`` acts
+#: on the target itself.
+_EXECUTOR_EXTENSION = {
+    SPARK_SQL_EXECUTOR: ".spark.sql",
+    SPARK_SQL_BATCH_EXECUTOR: ".spark-sql-batch.json",
+    SPARK_SCHEMA_EXECUTOR: ".schema.json",
+    SPARK_TABLE_EXECUTOR: ".spark-table.json",
+    TSQL_EXECUTOR: ".sql",
+    TSQL_BATCH_EXECUTOR: ".tsql-batch.json",
+    ALIAS_EXECUTOR: ".alias.json",
+    # A deployed file's payload is its exact bytes, whatever they are — a Python
+    # module, a generated statement — so the extension names the *role* rather
+    # than the content, which is the one thing every load file has in common.
+    LOAD_FILE_EXECUTOR: ".payload",
+}
+_PAYLOADLESS_EXECUTORS = frozenset(
+    {FOLDER_EXECUTOR, SQL_ENDPOINT_REFRESH_EXECUTOR}
+)
+#: Kinds that carry no payload even though their executor usually does. Only
+#: ``delete_file``: removing a deployed file needs the identity and nothing else,
+#: while writing one needs the exact bytes. Expressing it per *kind* keeps the
+#: strict requirement where it matters — a ``write_file`` with no payload is
+#: still rejected here rather than at install time.
+_PAYLOADLESS_KINDS = frozenset({DELETE_FILE})
+
+
+@dataclass(frozen=True)
+class BuildBundle:
+    """A validated bundle and the store holding its files.
+
+    The bundle store is deliberately independent of the target store. Inside
+    Fabric, payloads live on the session driver's temporary filesystem while
+    target Files mutations still use ``FabricStore``.
+    ``None`` remains accepted for compatibility with callers that reconstruct a
+    lightweight handle and let the installer use its environment store.
+    """
+
+    location: Location
+    plan: BuildPlan
+    store: Store | None = field(default=None, compare=False, repr=False)
+
+    @property
+    def bundle_id(self) -> str:
+        return self.plan.bundle_id
+
+
+# --- canonical form and identity --------------------------------------------
+
+
+def _canonical_bytes(mapping) -> bytes:
+    """A byte form that is identical for equal manifests on any platform."""
+
+    return json.dumps(
+        mapping, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    ).encode("utf-8")
+
+
+def compute_bundle_id(plan: BuildPlan) -> str:
+    """The identity of a plan, independent of its stored ``bundle_id`` field.
+
+    The field is blanked before hashing so a plan's id never depends on itself,
+    and everything else — signature, targets, sequences, payload hashes — feeds
+    in through the canonical mapping.
+    """
+
+    mapping = plan.to_mapping()
+    mapping["bundle_id"] = ""
+    return hashlib.sha256(_canonical_bytes(mapping)).hexdigest()
+
+
+def plan_to_yaml(plan: BuildPlan) -> str:
+    """The human-readable canonical manifest."""
+
+    return yaml.safe_dump(
+        plan.to_mapping(), sort_keys=False, default_flow_style=False, allow_unicode=True
+    )
+
+
+def plan_from_yaml(text: str) -> BuildPlan:
+    loaded = yaml.safe_load(text)
+    if not isinstance(loaded, dict):
+        raise BuildError("plan.yml must be a mapping")
+    try:
+        return BuildPlan.from_mapping(loaded)
+    except KeyError as exc:
+        raise BuildError(f"plan.yml is missing a required field: {exc}") from exc
+
+
+# --- writing -----------------------------------------------------------------
+
+
+def write_bundle(
+    location: Location,
+    *,
+    plan: BuildPlan,
+    payloads: Mapping[str, bytes],
+    store: Store,
+) -> BuildBundle:
+    """Write a bundle, manifest last, then reload and validate it.
+
+    ``payloads`` is keyed by each action's bundle-relative payload path.
+    """
+
+    for _, _, action in plan.actions():
+        if action.payload is None:
+            continue
+        _check_payload_path(action.payload)
+        if action.payload not in payloads:
+            raise BuildError(
+                f"action {action.id!r} references payload {action.payload!r} "
+                "but no payload was supplied for it"
+            )
+        digest = hashlib.sha256(payloads[action.payload]).hexdigest()
+        if action.payload_sha256 != digest:
+            raise BuildError(
+                f"action {action.id!r} payload hash does not match its content "
+                f"({action.payload_sha256} vs {digest})"
+            )
+
+    for relative, data in payloads.items():
+        store.write(location.join(*relative.split("/")), data)
+
+    # The manifest goes last: until it exists, the directory is not a bundle.
+    store.write(location.join(PLAN_FILENAME), plan_to_yaml(plan).encode("utf-8"))
+
+    return load_bundle(location, store=store)
+
+
+# --- loading and validation --------------------------------------------------
+
+
+def load_bundle(location: Location, *, store: Store) -> BuildBundle:
+    """Read a bundle and fully validate it before returning."""
+
+    plan_location = location.join(PLAN_FILENAME)
+    if not store.exists(plan_location):
+        raise BuildError(f"no bundle manifest at {plan_location.value}")
+
+    plan = plan_from_yaml(store.read(plan_location).decode("utf-8"))
+    validate_bundle(location, plan, store=store)
+    return BuildBundle(location=location, plan=plan, store=store)
+
+
+def validate_bundle(location: Location, plan: BuildPlan, *, store: Store) -> None:
+    """Reject any structural or integrity fault before an action runs.
+
+    Structure is checked first and needs no store, so a malformed manifest is
+    caught the same way whether or not its payloads happen to exist; payload
+    presence and hashes are checked second, against the store.
+    """
+
+    validate_plan_structure(plan)
+    _validate_payload_integrity(location, plan, store)
+
+
+def validate_plan_structure(plan: BuildPlan) -> None:
+    """Everything provable from the manifest alone, without reading payloads."""
+
+    if plan.format_version != SUPPORTED_FORMAT_VERSION:
+        raise BuildError(
+            f"unsupported bundle format version {plan.format_version} "
+            f"(this build supports {SUPPORTED_FORMAT_VERSION})"
+        )
+
+    for node in plan.omitted_nodes:
+        if node.reason not in OMISSION_REASONS:
+            raise BuildError(f"omitted node {node.node_id!r} has unknown reason {node.reason!r}")
+    omitted_ids = {node.node_id for node in plan.omitted_nodes}
+
+    target_ids = plan.target_ids
+    if len(target_ids) != len(plan.targets):
+        raise BuildError("duplicate target id in plan")
+    for target in plan.targets:
+        if (target.logical_item_type is None) != (target.logical_item_name is None):
+            raise BuildError(
+                f"target {target.id!r} carries an incomplete logical item identity"
+            )
+        if target.logical_item_type is not None:
+            expected = {
+                "Lakehouse": "lakehouse",
+                "Warehouse": "warehouse",
+            }.get(target.logical_item_type)
+            if expected != target.kind:
+                raise BuildError(
+                    f"target {target.id!r} binds logical {target.logical_item_type} "
+                    f"to physical kind {target.kind!r}"
+                )
+
+    seen_numbers: list[int] = []
+    batch_ids: set[str] = set()
+    action_ids: set[str] = set()
+
+    for sequence in plan.sequences:
+        seen_numbers.append(sequence.number)
+        for batch in sequence.batches:
+            if not batch.target_id:
+                raise BuildError(f"batch {batch.id!r} has no target")
+            if batch.target_id not in target_ids:
+                raise BuildError(
+                    f"batch {batch.id!r} names unknown target {batch.target_id!r}"
+                )
+            if batch.id in batch_ids:
+                raise BuildError(f"duplicate batch id {batch.id!r}")
+            batch_ids.add(batch.id)
+            for action in batch.actions:
+                if action.id in action_ids:
+                    raise BuildError(f"duplicate action id {action.id!r}")
+                action_ids.add(action.id)
+                _validate_action_shape(action, omitted_ids)
+
+    if seen_numbers != sorted(set(seen_numbers)) or len(seen_numbers) != len(set(seen_numbers)):
+        raise BuildError(
+            f"sequence numbers must be unique and ascending, got {seen_numbers}"
+        )
+
+
+def _validate_action_shape(action, omitted_ids) -> None:
+    if action.executor not in VALID_EXECUTORS:
+        raise BuildError(
+            f"action {action.id!r} uses unsupported executor {action.executor!r}"
+        )
+    if action.resource_node_id is not None and action.resource_node_id in omitted_ids:
+        raise BuildError(
+            f"action {action.id!r} targets omitted node {action.resource_node_id!r}"
+        )
+
+    if action.payload is None:
+        if action.payload_sha256 is not None:
+            raise BuildError(
+                f"action {action.id!r} has no payload but carries a payload hash"
+            )
+        if (
+            action.executor not in _PAYLOADLESS_EXECUTORS
+            and action.kind not in _PAYLOADLESS_KINDS
+        ):
+            raise BuildError(
+                f"action {action.id!r} uses executor {action.executor!r}, which needs a payload"
+            )
+        return
+
+    if action.executor in _PAYLOADLESS_EXECUTORS or action.kind in _PAYLOADLESS_KINDS:
+        raise BuildError(
+            f"action {action.id!r} is a {action.kind!r}, which takes no payload"
+        )
+    _check_payload_path(action.payload)
+    extension = _EXECUTOR_EXTENSION[action.executor]
+    if not action.payload.endswith(extension):
+        raise BuildError(
+            f"action {action.id!r} payload {action.payload!r} does not match "
+            f"executor {action.executor!r} extension {extension!r}"
+        )
+
+
+def _validate_payload_integrity(location, plan: BuildPlan, store: Store) -> None:
+    for _, _, action in plan.actions():
+        if action.payload is None:
+            continue
+        payload_location = location.join(*action.payload.split("/"))
+        if not store.exists(payload_location):
+            raise BuildError(f"action {action.id!r} payload is missing: {action.payload!r}")
+        digest = hashlib.sha256(store.read(payload_location)).hexdigest()
+        if digest != action.payload_sha256:
+            raise BuildError(
+                f"action {action.id!r} payload hash mismatch for {action.payload!r} "
+                f"(manifest {action.payload_sha256}, file {digest})"
+            )
+
+
+def _check_payload_path(payload: str) -> None:
+    _check_relative(payload, what="payload path")
+    if not payload.startswith(PAYLOAD_DIR + "/"):
+        raise BuildError(
+            f"payload {payload!r} must live under {PAYLOAD_DIR!r}/"
+        )
+
+
+def _check_relative(path: str, *, what: str) -> None:
+    if path.startswith("/") or ":" in path:
+        raise BuildError(f"{what} must be relative and stay in the bundle: {path!r}")
+    parts = path.split("/")
+    if any(part in ("", "..", ".") for part in parts):
+        raise BuildError(f"{what} must not be empty or traverse: {path!r}")

weaver/build_bundle/catalogue_actions.py

@@ -0,0 +1,275 @@
+"""Collect catalogue claims and render the three ordered catalogue barriers."""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from typing import Iterable, Mapping
+
+from ..catalogue.claims import CatalogueClaim, claim_rules_for_object_type
+from ..catalogue.state import Catalogue, for_targets, retaining
+from ..catalogue.reconcile import reconcile
+from ..catalogue.render import InstallationScope, identifier, literal
+from ..catalogue.state import Catalogue, for_targets, retaining
+from ..catalogue.tables import DICTIONARY_TABLES, REGISTRY, CatalogueTable
+from ..declaration.model import WeaverDocumentId
+from ..spark.tokens import object_token
+from .models import (
+    DELETE_CATALOGUE_CLAIMS,
+    PUBLISH_CATALOGUE,
+    PUBLISH_REGISTRY,
+    REFRESH_SQL_ENDPOINT,
+    BuildAction,
+    BuildBatch,
+)
+from .payloads import sha256_hex
+from .stages import CATALOGUE, PlannedStage
+
+
+def collect_claims(
+    catalogue: Catalogue,
+    identities: Iterable[WeaverDocumentId],
+    *,
+    stale_claims: Iterable[CatalogueClaim] = (),
+) -> tuple[CatalogueClaim, ...]:
+    """Every catalogue claim this build must delete before it does physical work.
+
+    Two sources, and they are symmetric: claims reconciliation already disproved
+    against the inventory, and claims held by the objects this build is about to
+    drop or remove. Both are passed in rather than one being read off the
+    catalogue, because a catalogue describes what is claimed — not which of those
+    claims some earlier step decided were wrong.
+    """
+
+    claims = list(stale_claims)
+    for identity in sorted(set(identities), key=str):
+        document = catalogue.registered.get(identity)
+        if document is None:
+            continue
+        tables = catalogue.rows.get(identity.item, {})
+        for rule in claim_rules_for_object_type(document.object_type):
+            if any(rule.owns(row, identity) for row in tables.get(rule.table.name, ())):
+                claims.append(CatalogueClaim(identity, rule))
+    return tuple(dict.fromkeys(claims))
+
+
+def _claim_statements(claims: Iterable[CatalogueClaim]) -> tuple[str, ...]:
+    grouped: dict[tuple[CatalogueTable, object], set[WeaverDocumentId]] = defaultdict(set)
+    for claim in claims:
+        grouped[(claim.rule.table, claim.identity.item)].add(claim.identity)
+
+    table_order = (REGISTRY, *reversed(DICTIONARY_TABLES))
+    statements = []
+    for table in table_order:
+        groups = sorted(
+            (
+                (item, identities)
+                for (claim_table, item), identities in grouped.items()
+                if claim_table == table
+            ),
+            key=lambda pair: str(pair[0]),
+        )
+        for item, identities in groups:
+            scope = InstallationScope(item.item_type, item.item_name)
+            predicates = []
+            for identity in sorted(identities, key=str):
+                rule = next(
+                    claim.rule
+                    for claim in claims
+                    if claim.identity == identity and claim.rule.table == table
+                )
+                values = rule.values(identity)
+                predicates.append(
+                    "("
+                    + " AND ".join(
+                        f"{identifier(column)} = {literal(value)}"
+                        for column, value in zip(
+                            rule.predicate_columns, values, strict=True
+                        )
+                    )
+                    + ")"
+                )
+            statements.append(
+                f"DELETE FROM {object_token('_', table.name)}\n"
+                f"WHERE {scope.predicate}\n  AND ("
+                + "\n    OR ".join(predicates)
+                + ")"
+            )
+    return tuple(statements)
+
+
+def _batch_payload(statements: Iterable[str]) -> bytes:
+    return (json.dumps(list(statements), indent=2, ensure_ascii=False) + "\n").encode(
+        "utf-8"
+    )
+
+
+def _stage(
+    *,
+    index: int,
+    slug: str,
+    description: str,
+    kind: str,
+    statements: Iterable[str],
+    control_target,
+) -> PlannedStage | None:
+    statements = tuple(statements)
+    if not statements:
+        return None
+    content = _batch_payload(statements)
+    filename = f"{slug}.spark-sql-batch.json"
+    action = BuildAction(
+        id=slug,
+        kind=kind,
+        resource_node_id=None,
+        executor="spark_sql_batch",
+        payload=filename,
+        payload_sha256=sha256_hex(content),
+    )
+    return PlannedStage(
+        phase=CATALOGUE,
+        index=index,
+        slug=slug,
+        description=description,
+        payloads={filename: content},
+        batches=(
+            BuildBatch(id=slug, target_id=control_target.id, actions=(action,)),
+        ),
+    )
+
+
+def render_catalogue_before_build(
+    catalogue: Catalogue,
+    identities: Iterable[WeaverDocumentId],
+    *,
+    control_target,
+    stale_claims: Iterable[CatalogueClaim] = (),
+) -> PlannedStage | None:
+    claims = collect_claims(catalogue, identities, stale_claims=stale_claims)
+    return _stage(
+        index=0,
+        slug="catalogue-before-build",
+        description="reconcile and remove catalogue claims before physical work",
+        kind=DELETE_CATALOGUE_CLAIMS,
+        statements=_claim_statements(claims),
+        control_target=control_target,
+    )
+
+
+def _item_signature(repository, item) -> str:
+    """The item's own signature, which is what an Installation row records."""
+
+    return next(
+        model.signature for model in repository.items if model.identity == item
+    )
+
+
+def render_catalogue_after_build(
+    repository,
+    selected_ids: Iterable[WeaverDocumentId],
+    target_by_item: Mapping,
+    *,
+    control_target,
+    current: Catalogue | None = None,
+) -> tuple[PlannedStage, ...]:
+    """Publish dictionaries and Installation in one batch, Registry last.
+
+    A final refresh of the Weaver Lakehouse's own SQL analytics endpoint closes
+    the build: the catalogue is a set of Delta tables like any other, and the next
+    reader of it — a report, a GUI, the next build — reaches it through that
+    endpoint.
+    """
+
+    from .. import __version__
+
+    selected_ids = set(selected_ids)
+
+    # The publication is a diff: what the repository describes, against what is
+    # persisted. Only the desired side drives the statements — see
+    # `CatalogueChanges` — but routing production through the same call the
+    # reporting uses is what stops the two drifting apart.
+    # Logical, then narrowed, then bound — in that order and visibly so. The
+    # narrowing is what keeps a Registry row meaning "this succeeded"; the
+    # binding is what lets an alias be certified as the thing it physically is.
+    logical = Catalogue.from_repository(repository)
+    certified = retaining(logical, repository, selected_ids)
+    desired = for_targets(
+        certified,
+        repository,
+        selected_ids,
+        {item: target.kind for item, target in target_by_item.items()},
+    )
+    binding_rows = {
+        item: (
+            {
+                "item_type": item.item_type,
+                "item_name": item.item_name,
+                "target_name": target_by_item[item].name,
+                "weaver_version": __version__,
+                "signature": _item_signature(repository, item),
+            },
+        )
+        for item in target_by_item
+    }
+    by_item = (current or Catalogue(rows={})).diff(desired).render_dml(
+        installation=binding_rows
+    )
+
+    catalogue_statements: list[str] = []
+    registry_statements: list[str] = []
+    for item in sorted(target_by_item, key=str):
+        result = by_item[item]
+        # Registry last, in its own barrier — taken from the structure rather
+        # than recovered from the SQL, so the ordering invariant is carried by
+        # the type instead of by a string match.
+        for table_plan in (*result.dictionaries, result.installation):
+            catalogue_statements.extend(table_plan.statements)
+        registry_statements.extend(result.registry.statements)
+
+    rendered = (
+        _stage(
+            index=1,
+            slug="publish-catalogue",
+            description="publish catalogue dictionaries and installations",
+            kind=PUBLISH_CATALOGUE,
+            statements=catalogue_statements,
+            control_target=control_target,
+        ),
+        _stage(
+            index=2,
+            slug="publish-registry",
+            description="publish item registry last",
+            kind=PUBLISH_REGISTRY,
+            statements=registry_statements,
+            control_target=control_target,
+        ),
+    )
+    published = tuple(stage for stage in rendered if stage is not None)
+    if not published:
+        return ()
+    return published + (_control_refresh_stage(control_target),)
+
+
+def _control_refresh_stage(control_target) -> PlannedStage:
+    return PlannedStage(
+        phase=CATALOGUE,
+        index=3,
+        slug="refresh-control-endpoint",
+        description="refresh the Weaver Lakehouse SQL endpoint after catalogue DML",
+        batches=(
+            BuildBatch(
+                id="refresh-control-endpoint",
+                target_id=control_target.id,
+                actions=(
+                    BuildAction(
+                        id="refresh-sql-endpoint-control",
+                        kind=REFRESH_SQL_ENDPOINT,
+                        resource_node_id=None,
+                        executor="sql_endpoint_refresh",
+                        payload=None,
+                        payload_sha256=None,
+                    ),
+                ),
+            ),
+        ),
+    )