weaverstack 0.1.1__py3-none-any.whl

weaver/catalogue/projection.py

@@ -0,0 +1,437 @@
+"""Projecting one logical item's catalogue rows from a validated declaration.
+
+This is the boundary the whole design turns on. On one side is a declaration
+that has already been read, validated and closed, and a build that has already
+decided which items it is installing. On the other side are rows. Nothing here
+re-reads a source file, imports an object module, or asks a physical table what
+shape it is — every value comes from the validated declaration or from the
+declaration's own resolved graph.
+
+**Only bound items are projected.** Objects owned by unbound items are *out of
+scope*, not deleted: a build has no opinion about an item it was not asked to
+install, and projecting them would invite a comparison that removed them.
+
+**Every row is stamped with the same item scope.** The scope is passed in once
+and applied to every row, rather than each projector deriving it — a projector
+that derived it differently would silently write into the wrong installation,
+which the renderer then refuses.
+
+**An alias is not a dependency.** A dependency row records the reference exactly
+as the author wrote it, and :data:`~weaver.catalogue.tables.ALIAS` records what
+the consuming item's alias points at. Joining Dependency, Alias and Registry is
+what yields the estate's whole graph; keeping them apart is what stops one item
+appearing to depend directly on another's physical object.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterable, Mapping
+
+from ..declaration.metadata import FOLDER, TABLE, VIEW, ObjectId, Reference
+from ..declaration.model import WeaverDocumentId, WeaverItemId, WeaverRepository
+from ..declaration.references import declared_column_notes, resolve_text
+from ..etl import PROCEDURE_TYPE, item_load_artefacts, load_artefacts_by_identity
+from .claims import catalogue_schema
+from .render import InstallationScope, Row, column_set
+from .tables import (
+    ALIAS,
+    CATALOGUE_TABLES,
+    COLUMN_DICTIONARY,
+    DEPENDENCY,
+    FOLDER_DICTIONARY,
+    FOREIGN_KEY_DICTIONARY,
+    INDEX_DICTIONARY,
+    INSTALLATION,
+    KEY_PRIMARY,
+    KEY_UNIQUE,
+    REGISTRY,
+    ROLE_DATA,
+    ROLE_LOAD,
+    SCHEMA_DICTIONARY,
+    TABLE_DICTIONARY,
+    CatalogueTable,
+)
+
+# Catalogue projection consumes the stable persisted target-kind vocabulary; it
+# does not depend on build-package binding classes. Keeping these values here
+# also prevents importing the build package while catalogue reconciliation is
+# still initialising.
+LAKEHOUSE_TARGET = "lakehouse"
+WAREHOUSE_TARGET = "warehouse"
+
+#: How an Weaver document kind names itself in the catalogue. Deliberately a translation
+#: rather than a reuse: Weaver document kinds are title case and the catalogue's vocabulary
+#: is lower case, and pinning the mapping here means a new Weaver document kind must be given
+#: a catalogue meaning rather than leaking one.
+OBJECT_TYPE_FOR_KIND = {FOLDER: "folder", TABLE: "table", VIEW: "view"}
+
+
+@dataclass(frozen=True)
+class CatalogueProjection:
+    """Every catalogue row one build invocation wants, for one installation."""
+
+    scope: InstallationScope
+    rows: Mapping[str, tuple[Row, ...]]
+
+    def for_table(self, table: CatalogueTable) -> tuple[Row, ...]:
+        return self.rows.get(table.name, ())
+
+    @property
+    def total(self) -> int:
+        return sum(len(rows) for rows in self.rows.values())
+
+
+def project_item_catalogue(
+    repository: WeaverRepository,
+    *,
+    item: WeaverItemId,
+    retained: Iterable[WeaverDocumentId],
+) -> CatalogueProjection:
+    """One item's catalogue rows, from the declaration and nothing else.
+
+    Every value here is a function of *source*: what the item declares, what it
+    aliases, what its documents describe. Nothing about a build, a binding or a
+    target reaches it. That is what makes the projection something a developer
+    keeps correct by adding a declaration, rather than a fixture someone has to
+    remember to update alongside one.
+
+    Not an *installation* projection, despite what this was once called. It says
+    what the repository declares; whether any of it has been installed, where,
+    and by which Weaver are separate facts composed at publication.
+
+    **No Registry row is written for an alias destination.** The Alias row —
+    this name points at that object — is a declaration and belongs here. The
+    Registry row is a certification that a physical object exists at that name
+    *and what it is*, and an alias is a view in a Warehouse and a table in a
+    Lakehouse. That cannot be answered without a binding, so it is not answered
+    here; see :func:`project_alias_registry`.
+    """
+
+    scope = InstallationScope(item.item_type, item.item_name)
+    retained = tuple(sorted(set(retained), key=str))
+    if any(identity.item != item for identity in retained):
+        raise ValueError(f"item projection {item} received a document owned elsewhere")
+
+    # ``retained`` carries every kind of registered object. Splitting them here
+    # rather than at the call site keeps the caller from having to know which is
+    # which — the repository already does.
+    alias_by_destination = {
+        alias.destination: alias
+        for alias in repository.aliases
+        if alias.destination.item == item
+    }
+    retained_aliases = tuple(
+        alias_by_destination[identity]
+        for identity in retained
+        if identity in alias_by_destination
+    )
+    loads = load_artefacts_by_identity(item_load_artefacts(repository, item=item))
+    retained_loads = tuple(
+        loads[identity] for identity in retained if identity in loads
+    )
+    retained = tuple(
+        identity
+        for identity in retained
+        if identity not in alias_by_destination and identity not in loads
+    )
+    documents = [repository.source_documents[identity] for identity in retained]
+    all_documents = tuple(repository.source_documents.values())
+    rows: dict[str, list[dict]] = {table.name: [] for table in CATALOGUE_TABLES}
+
+    for identity, source in zip(retained, documents):
+        common = _identity(scope, identity)
+        signature = source.effective_signature
+        rows[REGISTRY.name].append(
+            {
+                **common,
+                "object_type": OBJECT_TYPE_FOR_KIND[source.kind],
+                "object_role": ROLE_DATA,
+                "signature": signature,
+            }
+        )
+        described = _described(
+            source,
+            all_documents,
+            repository,
+        )
+        if source.kind == FOLDER:
+            rows[FOLDER_DICTIONARY.name].append(
+                {
+                    **common,
+                    **described,
+                    "file_key": column_set(source.document.file_keys),
+                    **_behaviour(source),
+                    "signature": signature,
+                }
+            )
+        else:
+            rows[TABLE_DICTIONARY.name].append(
+                {
+                    **common,
+                    "object_type": OBJECT_TYPE_FOR_KIND[source.kind],
+                    **described,
+                    "primary_key": column_set(source.document.primary_key),
+                    "not_null_columns": column_set(source.document.declared_not_null),
+                    "identity_column": source.document.identity,
+                    "comparison_columns": column_set(source.document.comparison_columns),
+                    **_behaviour(source),
+                    "signature": signature,
+                }
+            )
+        for column_name, note in declared_column_notes(source):
+            resolved = resolve_text(
+                note,
+                owner=source,
+                documents=all_documents,
+                aliases=repository.aliases,
+            )
+            rows[COLUMN_DICTIONARY.name].append(
+                {
+                    **common,
+                    "column_name": column_name,
+                    "description": resolved.literal,
+                    "description_reference": resolved.reference,
+                    "is_identity": column_name == source.document.identity,
+                    "signature": signature,
+                }
+            )
+        if source.document.primary_key:
+            rows[INDEX_DICTIONARY.name].append(
+                {
+                    **common,
+                    "index_type": KEY_PRIMARY,
+                    "column_set": column_set(source.document.primary_key),
+                    "signature": signature,
+                }
+            )
+        for unique in source.document.unique_keys:
+            rows[INDEX_DICTIONARY.name].append(
+                {
+                    **common,
+                    "index_type": KEY_UNIQUE,
+                    "column_set": column_set(unique),
+                    "signature": signature,
+                }
+            )
+        rows[FOREIGN_KEY_DICTIONARY.name].extend(
+            _foreign_keys(source, identity, common, signature)
+        )
+
+    # A load artefact claims the Registry and nothing else. It has no columns to
+    # describe, no keys to record and no dependencies to keep — it is a deployed
+    # module or a generated statement, and what the catalogue knows about it is
+    # that Weaver installed it and at what signature.
+    for artefact in retained_loads:
+        rows[REGISTRY.name].append(
+            {
+                **_identity(scope, artefact.identity),
+                "object_type": artefact.object_type,
+                "object_role": ROLE_LOAD,
+                "signature": artefact.signature,
+            }
+        )
+
+    retained_set = set(retained)
+    for edge in repository.dependency_edges:
+        if edge.consumer not in retained_set:
+            continue
+        source = repository.source_documents[edge.consumer]
+        rows[DEPENDENCY.name].append(
+            {
+                **_identity(scope, edge.consumer),
+                "dependency_name": edge.reference,
+                "is_within_item": edge.is_within_item,
+                "signature": source.effective_signature,
+            }
+        )
+
+    for alias in repository.aliases:
+        if alias.destination.item != item:
+            continue
+        rows[ALIAS.name].append(
+            {
+                **_scope(scope),
+                "destination_schema_name": _catalogue_schema(alias.destination),
+                "destination_object_name": alias.destination.object_id.object,
+                "source_item_type": alias.source.item.item_type,
+                "source_item_name": alias.source.item.item_name,
+                "source_schema_name": _catalogue_schema(alias.source),
+                "source_object_name": alias.source.object_id.object,
+                "signature": alias.signature,
+            }
+        )
+
+    used_schemas = sorted(
+        {
+            (_catalogue_schema(identity), identity.object_id.schema)
+            for identity in retained
+        }
+        | {
+            (_catalogue_schema(alias.destination), alias.destination.object_id.schema)
+            for alias in retained_aliases
+        }
+        # A generated load procedure puts a schema into use that no document
+        # declares an object in, so it would otherwise be a schema the
+        # installation uses and does not describe. A deployed file contributes
+        # nothing here: its schema half is a path, and the namespace it sits in
+        # is described by the folder document that owns the tree.
+        | {
+            (artefact.identity.object_id.schema, artefact.identity.object_id.schema)
+            for artefact in retained_loads
+            if artefact.object_type == PROCEDURE_TYPE
+        }
+    )
+    item_model = next(model for model in repository.items if model.identity == item)
+    schemas = {identity.schema: identity for identity in item_model.schemas}
+    for catalogue_name, declared_name in used_schemas:
+        schema = repository.schema_documents[schemas[declared_name]]
+        rows[SCHEMA_DICTIONARY.name].append(
+            {
+                **_scope(scope),
+                "schema_name": catalogue_name,
+                "description": schema.description,
+                "description_reference": None,
+                "signature": schema.source_hash,
+            }
+        )
+
+    return CatalogueProjection(
+        scope=scope,
+        rows={name: tuple(values) for name, values in rows.items()},
+    )
+
+
+def project_alias_registry(
+    repository: WeaverRepository,
+    *,
+    item: WeaverItemId,
+    retained: Iterable[WeaverDocumentId],
+    target_kind: str,
+) -> tuple[Row, ...]:
+    """Registry rows certifying this item's alias destinations, given a binding.
+
+    Separate from :func:`project_item_catalogue` because it is the one part of an
+    item's catalogue that cannot be derived from source: an alias is registered
+    as the thing it physically *is*, and that depends on what it was bound to.
+
+    Requiring the kind rather than defaulting it is deliberate. A default would
+    write a *wrong* Registry row quietly — a Warehouse alias recorded as a table —
+    and a wrong certification is the one failure the catalogue must never produce
+    on its own.
+    """
+
+    scope = InstallationScope(item.item_type, item.item_name)
+    wanted = set(retained)
+    return tuple(
+        {
+            **_identity(scope, alias.destination),
+            "object_type": _alias_object_type(alias.destination, target_kind),
+            "object_role": ROLE_DATA,
+            "signature": alias.signature,
+        }
+        for alias in sorted(
+            (
+                alias
+                for alias in repository.aliases
+                if alias.destination.item == item and alias.destination in wanted
+            ),
+            key=lambda alias: str(alias.destination),
+        )
+    )
+
+
+def _alias_object_type(destination: WeaverDocumentId, target_kind: str) -> str:
+    """What an alias destination physically *is*, in the catalogue's vocabulary.
+
+    Not a type of its own. An alias is registered as the thing it actually is —
+    a folder under ``Files``, a view in a Warehouse, a table in a Lakehouse —
+    because to every reader of the catalogue that is what it is, and because the
+    operations that matter (does it exist, how is it addressed, how is it
+    dropped) are the ordinary ones for that type. That a Lakehouse table alias
+    happens to be implemented as a OneLake shortcut is execution detail, the way
+    a managed table's storage layout is.
+
+    Its *alias-ness* is not lost: :data:`~weaver.catalogue.tables.ALIAS` records
+    it, and that is the only place that does.
+    """
+
+    if destination.is_files:
+        return OBJECT_TYPE_FOR_KIND[FOLDER]
+    if target_kind == WAREHOUSE_TARGET:
+        return OBJECT_TYPE_FOR_KIND[VIEW]
+    return OBJECT_TYPE_FOR_KIND[TABLE]
+
+
+def _scope(scope: InstallationScope) -> dict[str, str]:
+    return dict(scope.values)
+
+
+def _catalogue_schema(identity: WeaverDocumentId) -> str:
+    return catalogue_schema(identity)
+
+
+def _identity(scope: InstallationScope, identity: WeaverDocumentId) -> dict:
+    return {
+        **_scope(scope),
+        "schema_name": _catalogue_schema(identity),
+        "object_name": identity.object_id.object,
+    }
+
+
+def _described(source, all_documents, repository) -> dict:
+    description = resolve_text(
+        source.document.description,
+        owner=source,
+        documents=all_documents,
+        aliases=repository.aliases,
+    )
+    lineage = resolve_text(
+        source.document.lineage,
+        owner=source,
+        documents=all_documents,
+        aliases=repository.aliases,
+    )
+    return {
+        "description": description.literal,
+        "description_reference": description.reference,
+        "lineage": lineage.literal,
+        "lineage_reference": lineage.reference,
+    }
+
+
+def _behaviour(source) -> dict:
+    return {
+        "is_incremental": source.document.is_incremental,
+        "is_static": source.document.static,
+        "prohibit_rebuild": source.document.prohibit_rebuild,
+    }
+
+
+def _foreign_keys(source, identity, common, signature) -> list[dict]:
+    rows = []
+    for key in source.document.foreign_keys:
+        reference = key.logical_reference or Reference(
+            schema=key.reference.schema,
+            object=key.reference.object,
+        )
+        parent_item = (
+            WeaverItemId(reference.item_type, reference.item_name)
+            if reference.is_item_qualified
+            else identity.item
+        )
+        rows.append(
+            {
+                **common,
+                "column_set": column_set(key.columns),
+                "reference_item_type": parent_item.item_type,
+                "reference_item_name": parent_item.item_name,
+                "reference_schema_name": (
+                    f"Files/{reference.schema}" if reference.is_files else reference.schema
+                ),
+                "reference_object_name": reference.object,
+                "reference_column_set": column_set(key.reference_columns),
+                "signature": signature,
+            }
+        )
+    return rows

weaver/catalogue/reader.py

@@ -0,0 +1,152 @@
+"""Reading the catalogue through its fixed expected schema.
+
+A build compares what the catalogue holds with what the repository now declares,
+so it has to read tables that may not be there yet and may be an older shape than
+this Weaver knows about. Both are ordinary states, not faults:
+
+**Bootstrap.** The very first build has no catalogue at all — the tables are
+created by the build that then writes to them. A missing table reads as no rows.
+
+**Upgrade.** A Weaver that adds a column runs against a catalogue built by an
+older one. A missing column reads as a typed null, so newer code compares
+successfully against an older shape and the next build repairs it. An unexpected
+extra column is ignored, which is the other half: an older Weaver must not choke
+on a catalogue a newer one extended.
+
+What must **not** happen is a genuine failure being read as an empty catalogue. A
+permission error, a corrupt Delta log, an unavailable store or a broken session
+that returned "no rows" would make the next build's comparison conclude that
+everything is new — and, once drop policy lands, that everything the catalogue no
+longer mentions may be removed. So the absence check is deliberately narrow: only
+Spark's own ``TABLE_OR_VIEW_NOT_FOUND`` is absence. Everything else propagates.
+
+That asymmetry is the whole design of this module. Tolerance is cheap when it is
+specific and dangerous when it is a bare ``except``.
+
+**A read names the Lakehouse it reads from.** The catalogue lives in the Weaver
+Lakehouse; a build's other work is aimed at a destination Lakehouse; one session
+serves both. So a read takes a :class:`~weaver.spark.catalogue.SparkCatalogue`
+rather than a bare session — asking "the catalogue" of whatever the session is
+attached to would answer for the wrong Lakehouse, and answer *plausibly*, which
+is the failure mode this whole area exists to remove.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .render import InstallationScope, Row, identifier, qualified_name
+from .tables import CatalogueTable
+
+#: Spark's error class for a table or view that is not registered. A missing
+#: *schema* reports the same class, which is what we want — an installation whose
+#: schema `_` has never been created is as absent as one whose table has not.
+_ABSENT = frozenset({"TABLE_OR_VIEW_NOT_FOUND"})
+
+
+def _is_absent(exception: Exception) -> bool:
+    """Whether this exception means "not created yet" rather than "went wrong".
+
+    Keyed on Spark's error class rather than on message text, so a reworded
+    message cannot silently turn an infrastructure failure into an empty read. The
+    message is consulted only when no class is available, which is the case for a
+    session or connector that raises a plain error.
+    """
+
+    error_class = getattr(exception, "getErrorClass", None)
+    if callable(error_class):
+        try:
+            found = error_class()
+        except Exception:  # pragma: no cover - defensive; a broken accessor is not absence
+            found = None
+        if found:
+            return found in _ABSENT
+    return False
+
+
+def read_table(
+    catalogue: Any,
+    table: CatalogueTable,
+    *,
+    scope: InstallationScope | None = None,
+) -> tuple[Row, ...]:
+    """Every row of one catalogue table, projected through its expected schema.
+
+    ``catalogue`` is a :class:`~weaver.spark.catalogue.SparkCatalogue` bound to
+    the Weaver Lakehouse — the read has to say where the catalogue is, because
+    the session is not necessarily pointed at it.
+
+    ``scope`` narrows the read to one installation, which is what a build wants:
+    it compares and writes within one ``(repository, target_type)`` and has no
+    business seeing another's rows.
+
+    Returns plain dictionaries of ``str``/``bool``/``None`` — the same shape the
+    projection produces, so the two can be compared directly.
+    """
+
+    if catalogue is None:
+        raise ValueError(
+            f"reading {table.qualified} needs a Spark catalogue bound to the Weaver "
+            "Lakehouse — the catalogue lives there, and a session alone does not "
+            "say which Lakehouse that is"
+        )
+
+    name = catalogue.expand(qualified_name(table))
+    try:
+        existing = catalogue.spark.table(name).columns
+    except Exception as exception:
+        if _is_absent(exception):
+            return ()
+        raise
+
+    # Case-folded, because the local metastore lowercases column names where
+    # Fabric preserves them, and a column's presence must not depend on that.
+    present = {column.lower(): column for column in existing}
+
+    projected = ", ".join(
+        _projected_column(column, present.get(column.name.lower()))
+        for column in table.columns
+    )
+    where = ""
+    if scope is not None:
+        where = f" WHERE {scope.predicate}"
+
+    rows = catalogue.spark.sql(f"SELECT {projected} FROM {name}{where}").collect()
+    return tuple(row.asDict() for row in rows)
+
+
+def _projected_column(column, actual: str | None) -> str:
+    """One column of the expected schema, as a select expression.
+
+    Rendered as SQL rather than built with ``pyspark.sql.functions`` so this module
+    names no Spark API — the core stays importable without PySpark, and a session
+    is only ever duck-typed. It also keeps the projection inspectable as text.
+    """
+
+    if actual is None:
+        # Older shape: give this Weaver the column it expects, as a typed null.
+        return f"CAST(NULL AS {column.type.upper()}) AS {identifier(column.name)}"
+    # Cast even when present: an older catalogue may have stored a boolean as a
+    # string, and a comparison against a projected boolean would then differ for a
+    # row that has not actually changed.
+    return (
+        f"CAST({identifier(actual)} AS {column.type.upper()}) "
+        f"AS {identifier(column.name)}"
+    )
+
+
+def read_installation(
+    catalogue: Any, *, scope: InstallationScope, tables=None
+) -> dict[str, tuple[Row, ...]]:
+    """Every catalogue table, read for one installation.
+
+    Keyed by table name, so a caller compares table by table against the
+    projection without repeating the scope — which it could otherwise forget.
+    """
+
+    from .tables import CATALOGUE_TABLES
+
+    return {
+        table.name: read_table(catalogue, table, scope=scope)
+        for table in (tables if tables is not None else CATALOGUE_TABLES)
+    }