weaverstack 0.1.1__py3-none-any.whl

weaver/declaration/graph.py

@@ -0,0 +1,240 @@
+"""Dependency graph primitives.
+
+The graph knows nothing about what an edge *means*. That is deliberate,
+because there is more than one graph over the same objects:
+
+**Load order** follows every dependency. To load ``Reporting.OrderReport`` you
+first need the rows in ``Sales.Order``.
+
+**Build order** is nearly flat. Building a Folder is a directory; building a
+Delta table is a ``CREATE`` from its declared ``Schema`` — neither needs a
+single upstream object to exist. Only a Warehouse object has build
+dependencies, because its shape is inferred from its query. So a build is every
+Folder and every Delta table in one parallel wave, then the Warehouse objects in
+order, with a SQL endpoint refresh where the first of them reads Delta.
+
+Both are the same machinery over different edge sets, so this module takes
+nodes and edges and answers ordering questions about them.
+
+Order is deterministic: ties are broken by name, so the same repository always
+produces the same plan and two plans can be diffed.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Iterable, Mapping
+
+from ..errors import GraphError
+
+
+@dataclass(frozen=True)
+class Edge:
+    """``upstream`` must happen before ``downstream``."""
+
+    upstream: str
+    downstream: str
+
+    def __str__(self) -> str:
+        return f"{self.upstream} -> {self.downstream}"
+
+
+class Graph:
+    """A directed acyclic graph over named nodes."""
+
+    def __init__(self, nodes: Iterable[str], edges: Iterable[tuple[str, str]] = ()) -> None:
+        self._nodes = tuple(sorted(set(nodes)))
+        known = set(self._nodes)
+
+        seen: set[tuple[str, str]] = set()
+        collected: list[Edge] = []
+        for upstream, downstream in edges:
+            if upstream not in known:
+                raise GraphError(f"edge from unknown node {upstream!r}")
+            if downstream not in known:
+                raise GraphError(f"edge to unknown node {downstream!r}")
+            if upstream == downstream:
+                raise GraphError(f"{upstream} depends on itself")
+            if (upstream, downstream) in seen:
+                continue
+            seen.add((upstream, downstream))
+            collected.append(Edge(upstream=upstream, downstream=downstream))
+        self._edges = tuple(sorted(collected, key=lambda edge: (edge.upstream, edge.downstream)))
+
+        self._downstream: Mapping[str, list[str]] = defaultdict(list)
+        self._upstream: Mapping[str, list[str]] = defaultdict(list)
+        for edge in self._edges:
+            self._downstream[edge.upstream].append(edge.downstream)
+            self._upstream[edge.downstream].append(edge.upstream)
+
+        # Fail on construction: an unorderable graph is not a graph worth holding.
+        self._order = self._topological_order()
+
+    @property
+    def nodes(self) -> tuple[str, ...]:
+        return self._nodes
+
+    @property
+    def edges(self) -> tuple[Edge, ...]:
+        return self._edges
+
+    def __len__(self) -> int:
+        return len(self._nodes)
+
+    def __contains__(self, node: str) -> bool:
+        return node in set(self._nodes)
+
+    def _require(self, node: str) -> None:
+        if node not in set(self._nodes):
+            raise GraphError(f"unknown node: {node!r}")
+
+    def upstream_of(self, node: str) -> tuple[str, ...]:
+        """What this node depends on directly."""
+
+        self._require(node)
+        return tuple(sorted(self._upstream[node]))
+
+    def downstream_of(self, node: str) -> tuple[str, ...]:
+        """What depends on this node directly."""
+
+        self._require(node)
+        return tuple(sorted(self._downstream[node]))
+
+    def roots(self) -> tuple[str, ...]:
+        """Nodes that depend on nothing."""
+
+        return tuple(node for node in self._nodes if not self._upstream[node])
+
+    def leaves(self) -> tuple[str, ...]:
+        """Nodes nothing depends on."""
+
+        return tuple(node for node in self._nodes if not self._downstream[node])
+
+    # --- ordering ---------------------------------------------------------
+
+    def order(self) -> tuple[str, ...]:
+        """Every node, upstream before downstream, ties broken by name."""
+
+        return self._order
+
+    def layers(self) -> tuple[tuple[str, ...], ...]:
+        """Waves that may run in parallel.
+
+        Everything in a layer depends only on earlier layers, so a layer can be
+        dispatched together and joined before the next begins.
+        """
+
+        depth: dict[str, int] = {}
+        for node in self._order:
+            parents = self._upstream[node]
+            depth[node] = max((depth[parent] + 1 for parent in parents), default=0)
+
+        grouped: dict[int, list[str]] = defaultdict(list)
+        for node, level in depth.items():
+            grouped[level].append(node)
+        return tuple(tuple(sorted(grouped[level])) for level in sorted(grouped))
+
+    def _topological_order(self) -> tuple[str, ...]:
+        remaining = {node: len(self._upstream[node]) for node in self._nodes}
+        ready = sorted(node for node, count in remaining.items() if count == 0)
+        ordered: list[str] = []
+
+        while ready:
+            node = ready.pop(0)
+            ordered.append(node)
+            for child in sorted(self._downstream[node]):
+                remaining[child] -= 1
+                if remaining[child] == 0:
+                    ready.append(child)
+            ready.sort()
+
+        if len(ordered) != len(self._nodes):
+            cycle = self._find_cycle(set(self._nodes) - set(ordered))
+            raise GraphError("dependency cycle: " + " -> ".join(cycle))
+        return tuple(ordered)
+
+    def _find_cycle(self, candidates: set[str]) -> list[str]:
+        """One concrete cycle, so the message names the objects involved."""
+
+        path: list[str] = []
+        on_path: set[str] = set()
+        seen: set[str] = set()
+
+        def walk(node: str) -> list[str] | None:
+            if node in on_path:
+                return path[path.index(node):] + [node]
+            if node in seen:
+                return None
+            seen.add(node)
+            on_path.add(node)
+            path.append(node)
+            for child in sorted(self._downstream[node]):
+                if child not in candidates:
+                    continue
+                found = walk(child)
+                if found is not None:
+                    return found
+            path.pop()
+            on_path.discard(node)
+            return None
+
+        for start in sorted(candidates):
+            found = walk(start)
+            if found is not None:
+                return found
+        return sorted(candidates)
+
+    # --- traversal --------------------------------------------------------
+
+    def descendants(self, node: str) -> tuple[str, ...]:
+        """Everything reachable downstream, in dependency order.
+
+        This is what a rebuild must uncertify: an object whose upstream
+        definition is being rebuilt cannot stay certified.
+        """
+
+        return self._reach(node, self._downstream)
+
+    def ancestors(self, node: str) -> tuple[str, ...]:
+        """Everything reachable upstream, in dependency order."""
+
+        return self._reach(node, self._upstream)
+
+    def _reach(self, node: str, adjacency: Mapping[str, list[str]]) -> tuple[str, ...]:
+        self._require(node)
+        found: set[str] = set()
+        pending = list(adjacency[node])
+        while pending:
+            current = pending.pop()
+            if current in found:
+                continue
+            found.add(current)
+            pending.extend(adjacency[current])
+        return tuple(candidate for candidate in self._order if candidate in found)
+
+    def subgraph(
+        self,
+        selection: Iterable[str],
+        *,
+        with_ancestors: bool = False,
+        with_descendants: bool = False,
+    ) -> "Graph":
+        """A graph over a selection, optionally expanded along dependencies."""
+
+        chosen: set[str] = set()
+        for node in selection:
+            self._require(node)
+            chosen.add(node)
+            if with_ancestors:
+                chosen.update(self.ancestors(node))
+            if with_descendants:
+                chosen.update(self.descendants(node))
+        return Graph(
+            chosen,
+            [
+                (edge.upstream, edge.downstream)
+                for edge in self._edges
+                if edge.upstream in chosen and edge.downstream in chosen
+            ],
+        )

weaver/declaration/item_dependencies.py

@@ -0,0 +1,292 @@
+"""Item-owned dependency resolution and sparse logical projection."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Iterable, Mapping
+
+from ..errors import BuildError, DiscoveryError, GraphError
+from .graph import Graph
+from .metadata import ObjectId
+from .model import (
+    FILES,
+    ItemDependency,
+    RepositoryAlias,
+    WeaverDocumentId,
+    WeaverItemId,
+    WeaverRepository,
+)
+from .source import SourceDocument
+
+
+def resolve_item_dependencies(repository: WeaverRepository) -> WeaverRepository:
+    """Return ``repository`` with exact item-owned edges and a global DAG."""
+
+    native = repository.source_documents
+    aliases = {alias.destination: alias.source for alias in repository.aliases}
+    folded_native = {str(identity).casefold(): identity for identity in native}
+    folded_alias = {str(identity).casefold(): identity for identity in aliases}
+    edges: list[ItemDependency] = []
+    #: Graph edges, kept separately from ``edges`` because the two answer
+    #: different questions — see :func:`_document_graph`.
+    graph_edges: set[tuple[str, str]] = set()
+
+    for consumer, source in native.items():
+        if source.document.declares_dependencies:
+            references = tuple(
+                (dependency.qualified, WeaverDocumentId(consumer.item, dependency))
+                for dependency in source.document.dependencies
+            )
+        elif source.language == "python":
+            references = _python_references(source)
+        else:
+            references = []
+            for reference in source.discovered_references:
+                if reference.call:
+                    continue
+                if reference.is_qualified:
+                    edges.append(
+                        ItemDependency(
+                            consumer=consumer,
+                            reference=str(reference),
+                            resolution_kind="physical",
+                            is_within_item=False,
+                        )
+                    )
+                elif reference.object_id is not None:
+                    references.append(
+                        (
+                            str(reference),
+                            WeaverDocumentId(consumer.item, reference.object_id),
+                        )
+                    )
+
+        for written, destination in references:
+            producer, kind = _resolve_destination(
+                destination,
+                native=native,
+                aliases=aliases,
+                folded_native=folded_native,
+                folded_alias=folded_alias,
+                consumer=consumer,
+                written=written,
+            )
+            edges.append(
+                ItemDependency(
+                    consumer=consumer,
+                    producer=producer,
+                    reference=written,
+                    resolution_kind=kind,
+                    is_within_item=producer.item == consumer.item,
+                )
+            )
+            # ``destination`` is what the consumer named in its own namespace: the
+            # producer itself when that resolved natively, and the item's alias
+            # destination when it resolved through one. The edge above records the
+            # producer either way; the graph records the hop actually taken.
+            graph_edges.add((str(destination), str(consumer)))
+
+    unique = {
+        (edge.consumer, edge.reference, edge.producer, edge.resolution_kind): edge
+        for edge in edges
+    }
+    resolved = tuple(
+        sorted(
+            unique.values(),
+            key=lambda edge: (
+                str(edge.consumer),
+                edge.reference,
+                str(edge.producer) if edge.producer else "",
+            ),
+        )
+    )
+    graph = _document_graph(native, aliases, graph_edges)
+    item_graph = _item_graph(repository, resolved)
+    by_name = {str(item.identity): item.identity for item in repository.items}
+    return replace(
+        repository,
+        dependency_edges=resolved,
+        dependency_graph=graph,
+        item_graph=item_graph,
+        item_layers=tuple(
+            tuple(by_name[node] for node in layer) for layer in item_graph.layers()
+        ),
+    )
+
+
+def _document_graph(
+    native: Mapping[WeaverDocumentId, SourceDocument],
+    aliases: Mapping[WeaverDocumentId, WeaverDocumentId],
+    graph_edges: set[tuple[str, str]],
+) -> Graph:
+    """The graph incremental selection is planned against.
+
+    **This is deliberately not a projection of** :attr:`dependency_edges`. An
+    edge records what the author wrote and where it resolved to, and
+    :data:`~weaver.catalogue.tables.DEPENDENCY` publishes exactly that — an
+    alias edge names the *source document* as its producer, because that is the
+    truth about where the data comes from. The graph answers a different
+    question: what must be built, in what order. There the alias destination is
+    a thing in its own right — a shortcut or a view that some build has to
+    create — so the path is three hops rather than two:
+
+    .. code-block:: text
+
+        source document → alias destination → consumer document
+
+    Representing it that way is what lets impact propagate across items without
+    the planner needing a special case: an alias is an ordinary node, rebuilt
+    when its source is, and its consumers are ordinary descendants of it.
+
+    Every alias contributes its ``source → destination`` edge whether or not a
+    document consumes it. An alias with no consumer still has to be materialised
+    after the thing it points at exists.
+    """
+
+    edges = set(graph_edges)
+    for destination, source in aliases.items():
+        edges.add((str(source), str(destination)))
+    nodes = [str(identity) for identity in native]
+    nodes.extend(str(destination) for destination in aliases)
+    return Graph(nodes, sorted(edges))
+
+
+def _item_graph(repository: WeaverRepository, resolved: tuple[ItemDependency, ...]) -> Graph:
+    """The acyclic item-level graph a multi-item build is planned against.
+
+    One item depends on another when it reaches into it: either a document
+    resolves to a document that other item owns, or this item declares an alias
+    whose source lives there. The alias edge matters on its own — an alias with
+    no consumer yet still has to be materialised after its source exists — so it
+    is not left to be implied by the dependency edges.
+
+    Within-item edges are absent by construction: the document graph already
+    orders those, and an item cannot wait for itself.
+
+    A circular item graph is a **repository** fault. It is rejected here, while
+    the whole declaration is in view, rather than at the point some incremental
+    selection happens to exercise it — a repository whose items cannot be
+    ordered has no correct build, not merely no correct build today.
+    """
+
+    edges: set[tuple[str, str]] = set()
+    for edge in resolved:
+        if edge.producer is None or edge.producer.item == edge.consumer.item:
+            continue
+        edges.add((str(edge.producer.item), str(edge.consumer.item)))
+    for alias in repository.aliases:
+        # Repository parsing rejects a same-item alias, so every alias is an
+        # edge between two distinct items.
+        edges.add((str(alias.source.item), str(alias.destination.item)))
+
+    try:
+        return Graph(
+            (str(item.identity) for item in repository.items), sorted(edges)
+        )
+    except GraphError as exc:
+        raise GraphError(f"item {exc}") from exc
+
+
+def _resolve_destination(
+    destination: WeaverDocumentId,
+    *,
+    native: Mapping[WeaverDocumentId, SourceDocument],
+    aliases: Mapping[WeaverDocumentId, WeaverDocumentId],
+    folded_native: Mapping[str, WeaverDocumentId],
+    folded_alias: Mapping[str, WeaverDocumentId],
+    consumer: WeaverDocumentId,
+    written: str,
+) -> tuple[WeaverDocumentId, str]:
+    if destination in native:
+        return destination, "native"
+    if destination in aliases:
+        return aliases[destination], "alias"
+    case_match = folded_native.get(str(destination).casefold()) or folded_alias.get(
+        str(destination).casefold()
+    )
+    detail = f"; declared spelling is {case_match}" if case_match else ""
+    raise DiscoveryError(
+        f"{consumer}: dependency {written!r} does not resolve in item namespace{detail}"
+    )
+
+
+def _python_references(source: SourceDocument) -> list[tuple[str, WeaverDocumentId]]:
+    assert source.logical_id is not None
+    references: list[tuple[str, WeaverDocumentId]] = []
+    for imported in source.python_imports:
+        candidates = _resolved_python_modules(source.logical_id, imported)
+        for written, components in candidates:
+            if components and components[0] == "lib":
+                continue
+            is_files = bool(components and components[0] == FILES)
+            object_module = components[-1] if components else ""
+            parts = object_module.split("__")
+            if len(parts) != 2 or not all(parts):
+                continue
+            if len(components) != (2 if is_files else 1):
+                raise DiscoveryError(
+                    f"{source.node_id}: import {written!r} does not resolve to an "
+                    "item object or lib module"
+                )
+            references.append(
+                (
+                    written,
+                    WeaverDocumentId(
+                        source.logical_id.item,
+                        ObjectId(parts[0], parts[1]),
+                        is_files=is_files,
+                    ),
+                )
+            )
+    return references
+
+
+def _resolved_python_modules(logical_id: WeaverDocumentId, imported) -> list[tuple[str, tuple[str, ...]]]:
+    module = tuple(imported.module.split(".")) if imported.module else ()
+    if imported.level:
+        base = (FILES,) if logical_id.is_files else ()
+        parents = imported.level - 1
+        if parents > len(base):
+            raise DiscoveryError(
+                f"{logical_id}: import {imported} escapes the owning Weaver item"
+            )
+        resolved = base[: len(base) - parents] + module
+    else:
+        resolved = module
+
+    candidates = [(str(imported), resolved)]
+    if not module or not any("__" in component for component in module):
+        candidates = [
+            (f"{imported}.{name}".replace("..", "."), resolved + (name,))
+            for name in imported.names
+        ]
+    return candidates
+
+
+def project_bound_documents(
+    repository: WeaverRepository,
+    bound_items: Iterable[WeaverItemId],
+) -> tuple[SourceDocument, ...]:
+    """Select physical work by exact item only; do not pull in unbound ancestors."""
+
+    selected_items = set(bound_items)
+    if not selected_items:
+        raise BuildError("at least one Weaver item must be bound")
+    known_items = {item.identity for item in repository.items}
+    unknown = selected_items - known_items
+    if unknown:
+        raise BuildError(
+            "binding names item(s) absent from the repository: "
+            + ", ".join(sorted(map(str, unknown)))
+        )
+    selected = {
+        str(identity): source
+        for identity, source in repository.source_documents.items()
+        if identity.item in selected_items
+    }
+    order = (
+        repository.dependency_graph.order()
+        if repository.dependency_graph is not None
+        else tuple(sorted(selected))
+    )
+    return tuple(selected[node] for node in order if node in selected)