weaverstack 0.1.1__py3-none-any.whl

weaver/catalogue/reconcile.py

@@ -0,0 +1,231 @@
+"""Turning a projection into the statements a build appends, and nothing wider.
+
+Reconciliation for one installation is two statements per table: delete the rows
+this installation no longer projects, then merge the rows it does. Both are scoped
+to one ``(repository, target_type)``, so the reach of a whole build's catalogue
+work is bounded by construction rather than by care.
+
+**The statements do not depend on reading the catalogue first.** The delete keeps
+exactly the keys the projection claims and the merge is idempotent, so the pair is
+correct against any prior state — including a state the planner could not see.
+That is deliberate: a build that derived its deletes from an inventory would have
+its deletion scope widened by a failed read, which is the failure mode
+how-does-build-work §6 exists to prevent. Here a failed read cannot widen anything,
+because nothing is derived from it.
+
+Reading is still worth doing, for a different reason: a reviewer should be able to
+see what a bundle will change before it runs (§3, §17). :func:`compare` produces
+that summary — how many rows are new, changed, unchanged and removed — without any
+statement depending on it.
+
+**Ordering is the one strict invariant.** Dictionaries describe, Installation
+records the binding, Registry certifies. Registry is written last, so a row in it
+cannot outrun the work it attests to; the installer's barriers do the rest. Prune
+runs the order backwards — uncertify first, so nothing is left certified while its
+description is being removed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable, Mapping, Sequence
+
+from .projection import CatalogueProjection
+from .render import (
+    InstallationScope,
+    Row,
+    render_delete_obsolete,
+    render_delete_scope,
+    render_merge,
+)
+from .tables import (
+    CATALOGUE_TABLES,
+    DICTIONARY_TABLES,
+    INSTALLATION,
+    REGISTRY,
+    CatalogueTable,
+)
+
+@dataclass(frozen=True)
+class TableChanges:
+    """What reconciling one table would do. Reporting only — see the module note."""
+
+    table: CatalogueTable
+    inserted: int = 0
+    updated: int = 0
+    unchanged: int = 0
+    deleted: int = 0
+
+    @property
+    def touched(self) -> int:
+        return self.inserted + self.updated + self.deleted
+
+    @property
+    def is_noop(self) -> bool:
+        return self.touched == 0
+
+    def __str__(self) -> str:
+        return (
+            f"{self.table.name}: +{self.inserted} ~{self.updated} "
+            f"-{self.deleted} ={self.unchanged}"
+        )
+
+
+@dataclass(frozen=True)
+class TableReconciliation:
+    """One table's scoped statements, in the order they must run."""
+
+    table: CatalogueTable
+    #: None only for Installation, whose key *is* the installation scope: there is
+    #: at most one such row, so there is never an obsolete one to remove and the
+    #: merge alone keeps it current.
+    delete: str | None
+    #: None when the projection has no rows for this table — there is nothing to
+    #: merge, and an empty statement is worse than no action.
+    merge: str | None
+
+    @property
+    def statements(self) -> tuple[str, ...]:
+        return tuple(
+            statement for statement in (self.delete, self.merge) if statement is not None
+        )
+
+
+@dataclass(frozen=True)
+class CatalogueReconciliation:
+    """Every catalogue statement one build appends, grouped by when it may run.
+
+    The grouping is the contract: dictionaries may run in any order among
+    themselves, Installation follows them, and Registry follows everything. A
+    caller turns each group into its own barrier.
+    """
+
+    scope: InstallationScope
+    dictionaries: tuple[TableReconciliation, ...]
+    installation: TableReconciliation
+    registry: TableReconciliation
+
+    @property
+    def groups(self) -> tuple[tuple[str, tuple[TableReconciliation, ...]], ...]:
+        return (
+            ("reconcile catalogue dictionaries", self.dictionaries),
+            ("record the installation", (self.installation,)),
+            ("publish the registry", (self.registry,)),
+        )
+
+    @property
+    def statements(self) -> tuple[str, ...]:
+        """Every statement, in execution order. Registry's are last."""
+
+        return tuple(
+            statement
+            for _description, group in self.groups
+            for reconciliation in group
+            for statement in reconciliation.statements
+        )
+
+
+def reconcile(projection: CatalogueProjection) -> CatalogueReconciliation:
+    """The statements that make one installation's catalogue match its projection."""
+
+    scope = projection.scope
+    return CatalogueReconciliation(
+        scope=scope,
+        dictionaries=tuple(
+            _for_table(table, projection.for_table(table), scope)
+            for table in DICTIONARY_TABLES
+        ),
+        installation=_for_table(
+            INSTALLATION, projection.for_table(INSTALLATION), scope
+        ),
+        registry=_for_table(REGISTRY, projection.for_table(REGISTRY), scope),
+    )
+
+
+def _for_table(
+    table: CatalogueTable, rows: Sequence[Row], scope: InstallationScope
+) -> TableReconciliation:
+    return TableReconciliation(
+        table=table,
+        delete=render_delete_obsolete(table, rows, scope=scope),
+        merge=render_merge(table, rows, scope=scope),
+    )
+
+
+# --- what it would change ----------------------------------------------------
+
+
+def key_of(table: CatalogueTable, row: Row) -> tuple:
+    return tuple(row.get(name) for name in table.key)
+
+
+def _keyed(table: CatalogueTable, rows: Iterable[Row]) -> dict[tuple, Row]:
+    return {key_of(table, row): row for row in rows}
+
+
+def compare(
+    table: CatalogueTable, desired: Iterable[Row], existing: Iterable[Row]
+) -> TableChanges:
+    """How one table's rows differ from what is there — for review, not for DML.
+
+    A row is *unchanged* when every non-key column matches, which is exactly the
+    condition the merge's ``MATCHED`` guard tests. So a reported no-op is a real
+    no-op: the statement will run and write nothing.
+    """
+
+    wanted = _keyed(table, desired)
+    found = _keyed(table, existing)
+    inserted = updated = unchanged = 0
+    for key, row in wanted.items():
+        if key not in found:
+            inserted += 1
+        elif any(
+            row.get(name) != found[key].get(name) for name in table.comparison_columns
+        ):
+            updated += 1
+        else:
+            unchanged += 1
+    return TableChanges(
+        table=table,
+        inserted=inserted,
+        updated=updated,
+        unchanged=unchanged,
+        deleted=sum(1 for key in found if key not in wanted),
+    )
+
+
+def summarise(
+    projection: CatalogueProjection, existing: Mapping[str, Sequence[Row]]
+) -> tuple[TableChanges, ...]:
+    """What a build's catalogue work would change, table by table.
+
+    ``existing`` is keyed by table name, as :func:`weaver.catalogue.reader.
+    read_installation` returns it.
+    """
+
+    return tuple(
+        compare(table, projection.for_table(table), existing.get(table.name, ()))
+        for table in CATALOGUE_TABLES
+    )
+
+
+# --- the explicit prune scopes -----------------------------------------------
+
+
+def prune_installation(scope: InstallationScope) -> tuple[str, ...]:
+    """Remove one installation entirely, in dependency-safe order.
+
+    This is what decommissioning a target does. It is emphatically **not** what a
+    build does: a build that did not include a target type has no opinion about
+    it, which is a different thing from having removed it. Nothing in the build
+    path may reach this.
+
+    Registry goes first — uncertify before removing the descriptions, so no row is
+    ever left certified while what described it is gone.
+    """
+
+    # Uncertify first, remove dependent dictionaries next, and remove the
+    # installation root last. Delta does not enforce foreign keys, so this is
+    # the explicit ordered equivalent of ON DELETE CASCADE.
+    ordered = (REGISTRY, *reversed(DICTIONARY_TABLES), INSTALLATION)
+    return tuple(render_delete_scope(table, scope=scope) for table in ordered)

weaver/catalogue/render.py

@@ -0,0 +1,410 @@
+"""Rendering catalogue rows as deterministic, scoped Spark SQL.
+
+Every statement this module produces is frozen into a build bundle at generation
+time and executed unchanged (how-does-build-work §11, §12). Three properties follow,
+and each is enforced here rather than trusted to a caller:
+
+**Deterministic.** The same rows always render the same text, byte for byte. Rows
+are sorted by their key before rendering, so a mapping's iteration order cannot
+change a payload — and therefore cannot change a bundle's identity
+(how-does-build-work §15).
+
+**Scoped.** Every ``DELETE`` and every ``MERGE`` predicate names one
+``repository`` and one ``target_type``. A Lakehouse build physically cannot touch
+a Warehouse row, because the scope is not an argument a renderer might forget: it
+is part of the row's identity and part of every statement's ``WHERE``.
+
+**Explicit about values.** Every literal is cast to its declared column type, so
+a null is a typed null and a row of all-nulls cannot silently change the source
+frame's schema. Strings are escaped for Spark's default parser, where a backslash
+escapes.
+
+The one thing deliberately *not* frozen is the clock. ``current_timestamp()`` is
+rendered as a call, not as a literal: a rendered time would make the same input
+produce a different payload every run, which would destroy bundle identity for no
+gain. The engine supplies the instant; the payload stays stable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable, Mapping, Sequence
+
+from ..declaration.metadata import AUDIT_LIVE_DELETE_DATETIME
+from ..spark.tokens import EPOCH_TOKEN, object_token
+from .tables import (
+    AUDIT_DELETE_COLUMN,
+    AUDIT_INSERT_COLUMN,
+    AUDIT_UPDATE_COLUMN,
+    BOOLEAN,
+    CATALOGUE_SCHEMA,
+    ITEM_SCOPE_COLUMNS,
+    SCOPE_ITEM_NAME,
+    SCOPE_ITEM_TYPE,
+    TIMESTAMP,
+    CatalogueTable,
+)
+
+#: A row as projected: column name to value. Values are ``str``, ``bool`` or
+#: ``None`` — nothing needing a renderer of its own.
+Row = Mapping[str, object]
+
+
+@dataclass(frozen=True)
+class InstallationScope:
+    """The one installation a statement may touch.
+
+    Carried as a value rather than passed as two strings, so a renderer cannot be
+    called without it and a caller cannot supply half of it.
+    """
+
+    item_type: str
+    item_name: str
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        return ITEM_SCOPE_COLUMNS
+
+    @property
+    def values(self) -> Mapping[str, str]:
+        return {
+            SCOPE_ITEM_TYPE: self.item_type,
+            SCOPE_ITEM_NAME: self.item_name,
+        }
+
+    @property
+    def predicate(self) -> str:
+        return self.predicate_for()
+
+    def predicate_for(self, qualifier: str = "") -> str:
+        prefix = f"{qualifier}." if qualifier else ""
+        return " AND ".join(
+            f"{prefix}{identifier(column)} = {literal(value)}"
+            for column, value in self.values.items()
+        )
+
+    def owns(self, row: Row) -> bool:
+        return all(row.get(column) == value for column, value in self.values.items())
+
+    def __str__(self) -> str:
+        return f"{self.item_type}/{self.item_name}"
+
+
+def identifier(name: str) -> str:
+    """A back-tick quoted Spark identifier, safe for spaces and keywords."""
+
+    return "`" + name.replace("`", "``") + "`"
+
+
+def qualified_name(table: CatalogueTable) -> str:
+    """How a rendered statement names one catalogue table.
+
+    Not ``_.Registry``. The catalogue lives in the Weaver Lakehouse, and a build's
+    other statements are aimed at a destination Lakehouse — one session, two
+    places — so a name that resolved through the session's current catalogue
+    would put the record of the build wherever the session happened to be
+    pointed. The statement names the object; the batch names the Weaver
+    Lakehouse; the executor puts the two together (:mod:`weaver.spark.tokens`).
+    """
+
+    return object_token(CATALOGUE_SCHEMA, table.name)
+
+
+def literal(value: object) -> str:
+    """One value as a Spark SQL literal.
+
+    Spark's default parser treats a backslash as an escape, so both it and the
+    quote are escaped. Booleans and nulls are rendered as themselves rather than
+    as strings that happen to read that way.
+    """
+
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, str):
+        escaped = value.replace("\\", "\\\\").replace("'", "\\'")
+        return f"'{escaped}'"
+    if isinstance(value, (int, float)):
+        return repr(value)
+    raise TypeError(
+        f"catalogue values are strings, booleans or null, not {type(value).__name__}"
+    )
+
+
+def typed_literal(value: object, column_type: str) -> str:
+    """One value, cast to its declared type.
+
+    The cast is not decoration. A ``MERGE`` source is a ``SELECT`` union whose
+    schema comes from its first branch, so an uncast null would type a column by
+    accident and a later branch could then fail to match the target.
+    """
+
+    if column_type == BOOLEAN and value is not None and not isinstance(value, bool):
+        raise TypeError(f"expected a boolean for a {column_type} column, got {value!r}")
+    return f"CAST({literal(value)} AS {column_type.upper()})"
+
+
+def column_set(columns: Iterable[str]) -> str | None:
+    """A comma-separated column set, declared order preserved.
+
+    Order is meaning here — a key on ``(Region, Country)`` is not the same key as
+    one on ``(Country, Region)`` — so this never sorts. An empty set is null
+    rather than an empty string, because "no key" and "a key of no columns" are
+    different claims.
+    """
+
+    joined = ", ".join(columns)
+    return joined or None
+
+
+# --- statements ---------------------------------------------------------------
+
+
+def sorted_rows(table: CatalogueTable, rows: Iterable[Row]) -> tuple[Row, ...]:
+    """Rows in key order — the canonical order every statement renders in."""
+
+    def sort_key(row: Row) -> tuple[str, ...]:
+        return tuple(str(row.get(name) or "") for name in table.key)
+
+    return tuple(sorted(rows, key=sort_key))
+
+
+def render_merge(
+    table: CatalogueTable, rows: Sequence[Row], *, scope: InstallationScope
+) -> str | None:
+    """A scoped ``MERGE`` that inserts new rows and updates changed ones.
+
+    Returns None when there is nothing to merge, so a caller emits no action
+    rather than an empty statement.
+
+    An unchanged row is a genuine no-op: the ``MATCHED`` branch is guarded by a
+    null-safe comparison of every non-key column, so it neither writes nor
+    advances ``row_update_datetime``. That is what makes a rebuild of unchanged
+    Weaver document leave the catalogue alone.
+
+    **A published column is set on insert and never on update.** That is not an
+    optimisation, it is what makes the value mean what it claims. Every object a
+    build actually rebuilds reaches this statement as an *insert*: it is either
+    new, or its Registry claim was deleted before any physical work began. So an
+    update can only be a row whose projection changed while the object itself was
+    left alone — a document that forbids rebuilding is the case that exists — and
+    dating such a row to this build would say it was rebuilt when it was not.
+    Leaving published columns out of ``UPDATE SET`` keeps the old value, which is
+    the true one.
+    """
+
+    rows = sorted_rows(table, rows)
+    if not rows:
+        return None
+    _check_scope(table, rows, scope)
+    _check_unique_keys(table, rows)
+
+    columns = table.column_names
+    source = _source_relation(table, rows)
+
+    on = " AND ".join(
+        f"target.{identifier(name)} <=> source.{identifier(name)}" for name in table.key
+    )
+    # The target side is narrowed to this installation as well as matched on the
+    # key. The key already carries the scope, so this is belt and braces — and it
+    # is the belt that shows in a review.
+    scoped = scope.predicate_for("target")
+
+    comparison = table.comparison_columns
+    changed = " OR ".join(
+        f"NOT (target.{identifier(name)} <=> source.{identifier(name)})"
+        for name in comparison
+    )
+    updates = ", ".join(
+        [f"target.{identifier(name)} = source.{identifier(name)}" for name in comparison]
+        + [f"target.{identifier(AUDIT_UPDATE_COLUMN)} = current_timestamp()"]
+    )
+
+    # Named rather than positional: the audit columns are appended by the build in
+    # a fixed order, and pairing values to that order by position would put the
+    # sentinel in the wrong column the day the order changed.
+    supplied = {
+        AUDIT_INSERT_COLUMN: "current_timestamp()",
+        AUDIT_UPDATE_COLUMN: "current_timestamp()",
+        # A live row's delete datetime is a sentinel maximum, never null — all
+        # three audit columns are physically not null.
+        AUDIT_DELETE_COLUMN: (
+            f"CAST({literal(AUDIT_LIVE_DELETE_DATETIME)} AS {TIMESTAMP.upper()})"
+        ),
+    }
+    # A published column appears here and in *no* other clause. Not in the source
+    # relation, because no projection knows it; not in the comparison, because a
+    # value that is new every build would make every row differ; and not in the
+    # UPDATE, which is the substantive decision — see the note above the
+    # statement.
+    supplied.update(
+        {
+            name: f"CAST('{EPOCH_TOKEN}' AS {TIMESTAMP.upper()})"
+            for name in table.published_column_names
+        }
+    )
+    insert_columns = ", ".join(
+        identifier(name) for name in table.physical_columns
+    )
+    insert_values = ", ".join(
+        supplied[name] if name in supplied else f"source.{identifier(name)}"
+        for name in table.physical_columns
+    )
+
+    return (
+        f"MERGE INTO {qualified_name(table)} AS target\n"
+        f"USING (\n"
+        f"        {source}\n"
+        f") AS source\n"
+        f"   ON {scoped}\n"
+        f"  AND {on}\n"
+        f"WHEN MATCHED AND ({changed}) THEN UPDATE SET {updates}\n"
+        f"WHEN NOT MATCHED THEN INSERT ({insert_columns}) VALUES ({insert_values})\n"
+    )
+
+
+def _source_relation(table: CatalogueTable, rows: Sequence[Row]) -> str:
+    """The merge source: one ``VALUES`` relation, cast by an enclosing projection.
+
+    The obvious construction — one ``SELECT`` of cast literals per row, chained
+    with ``UNION ALL`` — does not scale, and the failure is nasty. Spark generates
+    Java for the plan, a method's bytecode may not exceed 64 KB, and a union of a
+    hundred projections exceeds it: the catalogue's own ``ColumnDictionary`` has a
+    row per column of every catalogue table, and that was enough to break the
+    bootstrap with ``Code grows beyond 64 KB``.
+
+    One ``VALUES`` relation is a single plan node however many rows it carries, so
+    the casts move outward into one projection over it. The values themselves are
+    bare literals: ``VALUES`` unifies a column's type across rows — all-null
+    becomes void — and the enclosing ``CAST`` settles it either way, which is what
+    keeps the source's schema exactly the target's.
+    """
+
+    tuples = ",\n                    ".join(
+        "(" + ", ".join(literal(row.get(name)) for name in table.column_names) + ")"
+        for row in rows
+    )
+    # Positional names for the raw relation, so a column called `repository` in the
+    # values cannot be confused with the aliased output of the same name.
+    raw = [f"c{index}" for index, _name in enumerate(table.column_names)]
+    projected = ", ".join(
+        f"CAST({identifier(raw[index])} AS {table.column(name).type.upper()})"
+        f" AS {identifier(name)}"
+        for index, name in enumerate(table.column_names)
+    )
+    names = ", ".join(identifier(name) for name in raw)
+    return (
+        f"SELECT {projected}\n"
+        f"          FROM VALUES\n"
+        f"                    {tuples}\n"
+        f"               AS source_values({names})"
+    )
+
+
+def render_delete_obsolete(
+    table: CatalogueTable, rows: Sequence[Row], *, scope: InstallationScope
+) -> str | None:
+    """A scoped ``DELETE`` of everything in this installation the rows do not claim.
+
+    An installation that now projects *no* rows for a table gets a plain scoped
+    delete: rendering nothing there would leave stale rows behind forever.
+
+    Returns None only for a table whose key is the installation scope itself —
+    :data:`~weaver.catalogue.tables.INSTALLATION`. There is at most one such row
+    per scope, so "the rows this projection does not claim" is empty by
+    construction and the merge alone keeps it current. Rendering a predicate over
+    no columns beyond the scope would delete the very row about to be merged.
+
+    The predicate is written as a disjunction of key equalities rather than a
+    tuple ``IN``: it renders identically on any engine, reads in a review, and
+    keeps the scope visible at the front of the statement.
+    """
+
+    rows = sorted_rows(table, rows)
+    _check_scope(table, rows, scope)
+    if not rows:
+        return f"DELETE FROM {qualified_name(table)}\n WHERE {scope.predicate}\n"
+
+    # Only the key columns beyond the scope: the scope is already in the WHERE.
+    identity = tuple(name for name in table.key if name not in scope.columns)
+    if not identity:
+        return None
+
+    keep = "\n           OR ".join(
+        "("
+        + " AND ".join(
+            f"{identifier(name)} <=> {typed_literal(row.get(name), table.column(name).type)}"
+            for name in identity
+        )
+        + ")"
+        for row in rows
+    )
+    return (
+        f"DELETE FROM {qualified_name(table)}\n"
+        f" WHERE {scope.predicate}\n"
+        f"   AND NOT (\n"
+        f"              {keep}\n"
+        f"           )\n"
+    )
+
+
+def render_delete_scope(table: CatalogueTable, *, scope: InstallationScope) -> str:
+    """A scoped ``DELETE`` of one whole installation from one table.
+
+    This is installation pruning: it is what decommissioning a target does, and it
+    is never what an ordinary build does. A build that did not include a target
+    type has no opinion about it, which is a different thing from having removed
+    it.
+    """
+
+    return f"DELETE FROM {qualified_name(table)}\n WHERE {scope.predicate}\n"
+
+
+
+
+def _check_unique_keys(table: CatalogueTable, rows: Sequence[Row]) -> None:
+    """Refuse a merge whose source holds two rows with one key.
+
+    Delta fails a ``MERGE`` when several source rows match one target row, and it
+    fails at *install* time — long after the bundle was reviewed. Catching it here
+    turns a late, obscure runtime error into a generation error naming the table
+    and the key, and it means a duplicate can only be a projection fault rather
+    than a mystery.
+    """
+
+    seen: dict[tuple, int] = {}
+    for row in rows:
+        key = tuple(row.get(name) for name in table.key)
+        seen[key] = seen.get(key, 0) + 1
+    duplicated = [key for key, count in seen.items() if count > 1]
+    if duplicated:
+        shown = "; ".join(
+            ", ".join(str(part) for part in key) for key in duplicated[:3]
+        )
+        raise ValueError(
+            f"{table.qualified}: {len(duplicated)} duplicated key(s) in the projected "
+            f"rows ({shown}) — a merge source must hold one row per key"
+        )
+
+
+def _check_scope(
+    table: CatalogueTable, rows: Iterable[Row], scope: InstallationScope
+) -> None:
+    """Refuse to render a statement over rows from another installation.
+
+    The guard exists because the consequence is invisible: a stray row would be
+    merged into the wrong installation's scope and read as truth. Cheap to check,
+    expensive to discover.
+    """
+
+    stray = [row for row in rows if not scope.owns(row)]
+    if stray:
+        found = ", ".join(
+            "/".join(repr(row.get(column)) for column in scope.columns)
+            for row in stray[:3]
+        )
+        raise ValueError(
+            f"{table.qualified}: {len(stray)} row(s) do not belong to installation "
+            f"{scope} ({found}) — a statement may only touch one installation"
+        )