nullcov 0.1.0__py3-none-any.whl

nullcov/__init__.py

@@ -0,0 +1,47 @@
+"""nullcov -- honest coverage for agent scaffolds.
+
+An agent test suite that reports 94% is usually reporting on the layers it
+happens to exercise. The layers nobody wrote a case for contribute nothing to
+the number and are absent from the summary, so the suite looks strongest exactly
+where it is blindest.
+
+nullcov reports those layers as ``unknown``. Never ``100%``, never omitted. And
+when production incidents are supplied, it separates the layers no case looks at
+from the layers whose cases pass while production fails anyway -- the second
+being the one that explains a green pipeline and a customer-visible outage on the
+same afternoon.
+"""
+
+from .alignment import (
+    AlignmentReport,
+    Finding,
+    FindingKind,
+    Incident,
+    align,
+)
+from .coverage import CoverageReport, SliceCoverage, Verdict
+from .plugin import case
+from .puremode import PureModeReport, PureModeViolation, is_active, pure_mode
+from .taxonomy import Layer, Taxonomy, TaxonomyError
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AlignmentReport",
+    "CoverageReport",
+    "Finding",
+    "FindingKind",
+    "Incident",
+    "Layer",
+    "PureModeReport",
+    "PureModeViolation",
+    "SliceCoverage",
+    "Taxonomy",
+    "TaxonomyError",
+    "Verdict",
+    "align",
+    "case",
+    "is_active",
+    "pure_mode",
+    "__version__",
+]

nullcov/alignment.py

@@ -0,0 +1,194 @@
+"""Reality alignment: does the suite look where the system actually bleeds?
+
+Half of surveyed enterprises shipped an agent that passed their own evaluations
+and still failed in front of a customer, and the most common complaint about
+those evaluations was poor alignment with real-world outcomes rather than
+insufficient coverage (VentureBeat VB Pulse, June 2026, n=157). Adding more
+cases does not fix that. The suite has to be checked against production.
+
+Cross-referencing incidents with coverage separates two failures that look
+identical on a dashboard:
+
+**Blind spot** -- incidents in a layer with no coverage. The suite never looks
+there. Adding cases is the fix.
+
+**Misalignment** -- incidents in a layer the suite reports as verified. The suite
+looks, and sees the wrong thing. More cases of the same shape make it worse, not
+better; the existing cases are the thing to distrust.
+
+Only the second one explains a green suite and an angry customer on the same day.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+from .coverage import CoverageReport, Verdict
+
+__all__ = [
+    "Incident",
+    "FindingKind",
+    "Finding",
+    "AlignmentReport",
+    "align",
+]
+
+
+@dataclass(frozen=True)
+class Incident:
+    """One production failure, attributed to a layer.
+
+    Attribution is the input this engine cannot derive for itself. It comes from
+    an incident review, a postmortem tag, or a trace whose failing span maps onto
+    a layer. Attribution quality bounds the quality of everything below.
+    """
+
+    incident_id: str
+    layer: str
+    summary: str = ""
+    severity: str = "unknown"
+    occurred_at: datetime | None = None
+
+    def __post_init__(self) -> None:
+        if not self.incident_id:
+            raise ValueError("incident_id cannot be empty")
+        if not self.layer:
+            raise ValueError(f"incident {self.incident_id} has no layer attribution")
+
+
+class FindingKind(str, Enum):
+    BLIND_SPOT = "blind_spot"
+    MISALIGNMENT = "misalignment"
+    UNATTRIBUTED = "unattributed"
+
+
+@dataclass(frozen=True)
+class Finding:
+    """A layer where production and the test suite disagree."""
+
+    kind: FindingKind
+    layer: str
+    incident_count: int
+    incident_share: float
+    coverage_verdict: Verdict
+    cases_run: int
+
+    def explain(self) -> str:
+        share = f"{self.incident_share:.0%}"
+        if self.kind is FindingKind.BLIND_SPOT:
+            return (
+                f"{self.layer}: {self.incident_count} incidents ({share} of all "
+                f"attributed) and no coverage at all. The suite cannot speak to "
+                f"this layer."
+            )
+        if self.kind is FindingKind.MISALIGNMENT:
+            return (
+                f"{self.layer}: {self.incident_count} incidents ({share} of all "
+                f"attributed) despite {self.cases_run} passing cases. The cases "
+                f"pass and production still fails -- distrust these cases before "
+                f"adding more."
+            )
+        return (
+            f"{self.layer}: {self.incident_count} incidents ({share} of all "
+            f"attributed) against a layer that is not in the declared taxonomy."
+        )
+
+
+@dataclass
+class AlignmentReport:
+    """Findings, ordered by how much production pain sits behind each."""
+
+    findings: list[Finding] = field(default_factory=list)
+    total_incidents: int = 0
+    layers_without_incidents: frozenset[str] = field(default_factory=frozenset)
+
+    def of_kind(self, kind: FindingKind) -> list[Finding]:
+        return [f for f in self.findings if f.kind is kind]
+
+    @property
+    def blind_spots(self) -> list[Finding]:
+        return self.of_kind(FindingKind.BLIND_SPOT)
+
+    @property
+    def misalignments(self) -> list[Finding]:
+        return self.of_kind(FindingKind.MISALIGNMENT)
+
+    def to_dict(self) -> dict:
+        return {
+            "total_incidents": self.total_incidents,
+            "layers_without_incidents": sorted(self.layers_without_incidents),
+            "findings": [
+                {
+                    "kind": f.kind.value,
+                    "layer": f.layer,
+                    "incident_count": f.incident_count,
+                    "incident_share": round(f.incident_share, 4),
+                    "coverage_verdict": f.coverage_verdict.value,
+                    "cases_run": f.cases_run,
+                    "explanation": f.explain(),
+                }
+                for f in self.findings
+            ],
+        }
+
+
+def align(coverage: CoverageReport, incidents: list[Incident]) -> AlignmentReport:
+    """Cross-reference production incidents against coverage.
+
+    Findings are sorted by incident count, so the layer costing the most in
+    production is read first. Layers with incidents but a clean, exercised
+    suite are reported as misalignments rather than passes -- a layer that keeps
+    failing in front of users has not been verified in any sense that matters,
+    whatever the suite says.
+    """
+    if not incidents:
+        return AlignmentReport(
+            findings=[],
+            total_incidents=0,
+            layers_without_incidents=coverage.declared_layers,
+        )
+
+    counts = Counter(incident.layer for incident in incidents)
+    total = len(incidents)
+    cases_by_layer = {
+        layer: sum(s.cases_run for s in coverage.slices if s.layer == layer)
+        for layer in {s.layer for s in coverage.slices}
+    }
+
+    findings: list[Finding] = []
+    for layer, count in counts.items():
+        share = count / total
+        cases_run = cases_by_layer.get(layer, 0)
+
+        if layer not in coverage.declared_layers:
+            kind = FindingKind.UNATTRIBUTED
+            verdict = Verdict.UNKNOWN
+        else:
+            verdict = coverage.layer_verdict(layer)
+            kind = (
+                FindingKind.BLIND_SPOT
+                if verdict is Verdict.UNKNOWN
+                else FindingKind.MISALIGNMENT
+            )
+
+        findings.append(
+            Finding(
+                kind=kind,
+                layer=layer,
+                incident_count=count,
+                incident_share=share,
+                coverage_verdict=verdict,
+                cases_run=cases_run,
+            )
+        )
+
+    findings.sort(key=lambda f: (-f.incident_count, f.layer))
+
+    return AlignmentReport(
+        findings=findings,
+        total_incidents=total,
+        layers_without_incidents=coverage.declared_layers - set(counts),
+    )

nullcov/coverage.py

@@ -0,0 +1,200 @@
+"""The honesty engine.
+
+Conventional coverage collapses to a single percentage. That percentage is a lie
+whenever a layer of the system was never exercised at all: zero cases divided by
+zero cases is reported as "fine", and a green aggregate hides an untested layer.
+
+nullcov refuses that. Coverage here is three-valued. A layer that was never
+exercised is ``UNKNOWN`` -- never ``100%``, never silently dropped from the
+average. The aggregate carries the unknowns with it, so a summary cannot launder
+a blind spot.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+__all__ = [
+    "Verdict",
+    "SliceCoverage",
+    "CoverageReport",
+]
+
+
+class Verdict(str, Enum):
+    """Outcome for a single slice.
+
+    ``UNKNOWN`` is the whole point of this library: it is structurally distinct
+    from ``VERIFIED`` and never coerces to a passing number.
+    """
+
+    VERIFIED = "verified"
+    FAILED = "failed"
+    UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class SliceCoverage:
+    """Coverage for one slice -- one contract of one layer.
+
+    A slice is the smallest unit that can be honestly reported on. ``cases_run``
+    counts assertions that actually executed; a slice that was declared but never
+    executed has ``cases_run == 0`` and is ``UNKNOWN``.
+    """
+
+    layer: str
+    slice_id: str
+    cases_run: int = 0
+    cases_failed: int = 0
+
+    def __post_init__(self) -> None:
+        if self.cases_run < 0 or self.cases_failed < 0:
+            raise ValueError("case counts cannot be negative")
+        if self.cases_failed > self.cases_run:
+            raise ValueError(
+                f"{self.layer}/{self.slice_id}: cases_failed ({self.cases_failed}) "
+                f"exceeds cases_run ({self.cases_run})"
+            )
+
+    @property
+    def verdict(self) -> Verdict:
+        if self.cases_run == 0:
+            return Verdict.UNKNOWN
+        if self.cases_failed > 0:
+            return Verdict.FAILED
+        return Verdict.VERIFIED
+
+    @property
+    def cases_passed(self) -> int:
+        return self.cases_run - self.cases_failed
+
+    def merge(self, other: SliceCoverage) -> SliceCoverage:
+        """Combine two observations of the same slice."""
+        if (self.layer, self.slice_id) != (other.layer, other.slice_id):
+            raise ValueError("cannot merge coverage for different slices")
+        return SliceCoverage(
+            layer=self.layer,
+            slice_id=self.slice_id,
+            cases_run=self.cases_run + other.cases_run,
+            cases_failed=self.cases_failed + other.cases_failed,
+        )
+
+
+@dataclass
+class CoverageReport:
+    """Aggregate coverage that keeps its unknowns.
+
+    ``declared_layers`` is the taxonomy the author committed to up front. It is
+    what makes blind spots visible at all: without a declaration of what *should*
+    exist, an unexercised layer is indistinguishable from a layer that does not
+    exist, and no tool can tell you the difference.
+    """
+
+    declared_layers: frozenset[str] = field(default_factory=frozenset)
+    slices: list[SliceCoverage] = field(default_factory=list)
+
+    def add(self, slice_coverage: SliceCoverage) -> None:
+        """Record a slice, merging into an existing entry if already present."""
+        key = (slice_coverage.layer, slice_coverage.slice_id)
+        for index, existing in enumerate(self.slices):
+            if (existing.layer, existing.slice_id) == key:
+                self.slices[index] = existing.merge(slice_coverage)
+                return
+        self.slices.append(slice_coverage)
+
+    # -- layer-level views -------------------------------------------------
+
+    def exercised_layers(self) -> frozenset[str]:
+        """Layers with at least one case that actually ran."""
+        return frozenset(s.layer for s in self.slices if s.cases_run > 0)
+
+    def unknown_layers(self) -> frozenset[str]:
+        """Declared layers that no case ever exercised.
+
+        These are the blind spots. They are reported explicitly and are never
+        folded into a pass percentage.
+        """
+        return self.declared_layers - self.exercised_layers()
+
+    def undeclared_layers(self) -> frozenset[str]:
+        """Layers that produced cases but were never declared in the taxonomy.
+
+        Usually a typo in a layer name, or drift between the taxonomy and the
+        tests. Either way the author should see it rather than have it silently
+        counted as coverage.
+        """
+        return self.exercised_layers() - self.declared_layers
+
+    def layer_verdict(self, layer: str) -> Verdict:
+        """Worst-case verdict across a layer's slices.
+
+        A layer is only ``VERIFIED`` when every one of its slices ran clean. One
+        failing slice makes the layer ``FAILED``; no cases at all make it
+        ``UNKNOWN``.
+        """
+        layer_slices = [s for s in self.slices if s.layer == layer]
+        if not layer_slices or all(s.cases_run == 0 for s in layer_slices):
+            return Verdict.UNKNOWN
+        if any(s.verdict is Verdict.FAILED for s in layer_slices):
+            return Verdict.FAILED
+        return Verdict.VERIFIED
+
+    def slices_by_verdict(self, verdict: Verdict) -> list[SliceCoverage]:
+        return [s for s in self.slices if s.verdict is verdict]
+
+    # -- aggregate ---------------------------------------------------------
+
+    @property
+    def total_cases(self) -> int:
+        return sum(s.cases_run for s in self.slices)
+
+    @property
+    def total_failed(self) -> int:
+        return sum(s.cases_failed for s in self.slices)
+
+    def verified_case_rate(self) -> float | None:
+        """Pass rate over cases that actually ran, or ``None`` if none ran.
+
+        This deliberately returns ``None`` rather than ``1.0`` for an empty run.
+        It is also *not* the headline number: it says nothing about the layers in
+        :meth:`unknown_layers`, which is exactly the number people misread.
+        """
+        if self.total_cases == 0:
+            return None
+        return (self.total_cases - self.total_failed) / self.total_cases
+
+    def is_honest_pass(self) -> bool:
+        """True only when nothing failed *and* nothing is unknown.
+
+        This is the gate condition. A run with unknown layers is not a pass,
+        because the suite cannot speak to those layers at all.
+        """
+        return (
+            self.total_failed == 0
+            and not self.unknown_layers()
+            and not self.undeclared_layers()
+            and self.total_cases > 0
+        )
+
+    def to_dict(self) -> dict:
+        """Machine-readable form, for CI artifacts and the alignment engine."""
+        return {
+            "declared_layers": sorted(self.declared_layers),
+            "unknown_layers": sorted(self.unknown_layers()),
+            "undeclared_layers": sorted(self.undeclared_layers()),
+            "total_cases": self.total_cases,
+            "total_failed": self.total_failed,
+            "verified_case_rate": self.verified_case_rate(),
+            "honest_pass": self.is_honest_pass(),
+            "slices": [
+                {
+                    "layer": s.layer,
+                    "slice_id": s.slice_id,
+                    "cases_run": s.cases_run,
+                    "cases_failed": s.cases_failed,
+                    "verdict": s.verdict.value,
+                }
+                for s in sorted(self.slices, key=lambda s: (s.layer, s.slice_id))
+            ],
+        }

nullcov/langgraph/__init__.py

@@ -0,0 +1,26 @@
+"""LangGraph-specific structural coverage.
+
+Separate from the framework-agnostic core (``nullcov.coverage``,
+``nullcov.alignment``) on purpose: this module reads ground truth out of a
+compiled ``langgraph`` graph object, which is inherently framework-specific.
+Import it only if you have ``langgraph`` installed (``pip install
+nullcov[langgraph]``); the rest of the package does not depend on it.
+"""
+
+from .topology import (
+    Branch,
+    BranchCoverage,
+    CoverageAccumulator,
+    GraphTopology,
+    RunTrace,
+    trace_run,
+)
+
+__all__ = [
+    "Branch",
+    "BranchCoverage",
+    "CoverageAccumulator",
+    "GraphTopology",
+    "RunTrace",
+    "trace_run",
+]

nullcov/langgraph/topology.py

@@ -0,0 +1,208 @@
+"""Structural coverage for LangGraph state machines.
+
+Every other layer in this library depends on a human typing a truthful string:
+``@nullcov.case(layer="escalation", ...)``. Nothing stops that string from
+drifting from the code -- a developer adds a node to the graph, forgets to add
+a case for it, and it is invisible everywhere: not in the declared taxonomy, not
+in the executed cases, not reported as anything at all. It is not even a
+``[????]`` -- it just does not exist as far as nullcov is concerned.
+
+This module removes the human from that step for one specific, checkable claim:
+which nodes and which conditional branches of a *compiled LangGraph graph* were
+actually traversed by the test suite, aggregated across every recorded run.
+
+The ground truth is the compiled graph object itself, via
+``CompiledGraph.get_graph()`` -- not a file a developer maintains by hand. A new
+node or a new conditional destination is a blind spot the moment it compiles,
+before anyone writes a test for it, because the topology is read from the
+artifact under test rather than declared about it.
+
+This is deliberately narrow. It answers one question precisely -- "of all the
+routing decisions this graph can make, which ones has any test ever forced it
+to make?" -- and answers it exactly, with no LLM judgment involved. It does not
+replace the taxonomy/pure-mode/alignment machinery elsewhere in this package;
+it is a second, independent source of ground truth for the one framework where
+that ground truth happens to be mechanically extractable.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+__all__ = [
+    "Branch",
+    "GraphTopology",
+    "RunTrace",
+    "trace_run",
+    "BranchCoverage",
+    "CoverageAccumulator",
+]
+
+_START = "__start__"
+_END = "__end__"
+
+
+class _StreamableGraph(Protocol):
+    def get_graph(self) -> Any: ...
+    def stream(self, input: Any, stream_mode: str) -> Any: ...
+
+
+@dataclass(frozen=True)
+class Branch:
+    """One possible transition out of a conditional routing node."""
+
+    source: str
+    target: str
+
+
+@dataclass(frozen=True)
+class GraphTopology:
+    """The routing structure of a compiled graph, as it actually compiled.
+
+    ``__start__`` and ``__end__`` are LangGraph's own bookkeeping nodes, not
+    agent logic, and are excluded from both ``nodes`` and ``branches`` -- there
+    is no meaningful sense in which the entry point is "undertested".
+    """
+
+    nodes: frozenset[str]
+    branches: frozenset[Branch]
+
+    @classmethod
+    def extract(cls, compiled_graph: _StreamableGraph) -> GraphTopology:
+        """Read the topology straight off a compiled graph.
+
+        Every outgoing edge from a node with more than one destination is
+        treated as a conditional branch -- this covers LangGraph's own
+        ``conditional=True`` marking, and also catches a node that fans out to
+        multiple unconditional edges, which is the same coverage question in
+        different syntax.
+        """
+        graph = compiled_graph.get_graph()
+
+        nodes = frozenset(
+            name for name in graph.nodes if name not in (_START, _END)
+        )
+
+        destinations_by_source: dict[str, set[str]] = {}
+        for edge in graph.edges:
+            if edge.source == _START:
+                continue
+            destinations_by_source.setdefault(edge.source, set()).add(edge.target)
+
+        branches: set[Branch] = set()
+        for source, destinations in destinations_by_source.items():
+            if len(destinations) > 1:
+                branches.update(Branch(source, target) for target in destinations)
+
+        return cls(nodes=nodes, branches=branches)
+
+    def branches_from(self, source: str) -> frozenset[Branch]:
+        return frozenset(b for b in self.branches if b.source == source)
+
+    @property
+    def branching_nodes(self) -> frozenset[str]:
+        return frozenset(b.source for b in self.branches)
+
+
+@dataclass(frozen=True)
+class RunTrace:
+    """The path one execution actually took."""
+
+    nodes_visited: tuple[str, ...]
+
+    @property
+    def edges_taken(self) -> frozenset[Branch]:
+        return frozenset(
+            Branch(a, b) for a, b in zip(self.nodes_visited, self.nodes_visited[1:])
+        )
+
+
+def trace_run(compiled_graph: _StreamableGraph, input: Any) -> RunTrace:
+    """Run a graph once and record exactly which nodes it visited, in order.
+
+    Uses ``stream_mode="updates"``, which yields one ``{node_name: delta}``
+    chunk per step -- the same mechanism LangGraph itself uses to report
+    progress, so this adds no instrumentation of its own that could drift from
+    what actually executed.
+    """
+    visited: list[str] = []
+    for chunk in compiled_graph.stream(input, stream_mode="updates"):
+        visited.extend(chunk.keys())
+    return RunTrace(nodes_visited=tuple(visited))
+
+
+@dataclass(frozen=True)
+class BranchCoverage:
+    """Coverage of one graph's topology across every recorded run."""
+
+    topology: GraphTopology
+    nodes_visited: frozenset[str]
+    branches_taken: frozenset[Branch]
+
+    @property
+    def unvisited_nodes(self) -> frozenset[str]:
+        return self.topology.nodes - self.nodes_visited
+
+    @property
+    def untaken_branches(self) -> frozenset[Branch]:
+        return self.topology.branches - self.branches_taken
+
+    def branch_rate(self) -> float | None:
+        total = len(self.topology.branches)
+        if total == 0:
+            return None
+        return len(self.branches_taken) / total
+
+    def is_fully_covered(self) -> bool:
+        return not self.unvisited_nodes and not self.untaken_branches
+
+    def to_dict(self) -> dict:
+        return {
+            "nodes_total": sorted(self.topology.nodes),
+            "nodes_visited": sorted(self.nodes_visited),
+            "unvisited_nodes": sorted(self.unvisited_nodes),
+            "branches_total": [
+                {"source": b.source, "target": b.target}
+                for b in sorted(self.topology.branches, key=lambda b: (b.source, b.target))
+            ],
+            "untaken_branches": [
+                {"source": b.source, "target": b.target}
+                for b in sorted(self.untaken_branches, key=lambda b: (b.source, b.target))
+            ],
+            "branch_rate": self.branch_rate(),
+            "fully_covered": self.is_fully_covered(),
+        }
+
+
+@dataclass
+class CoverageAccumulator:
+    """Collects traces from many test cases against one topology.
+
+    One accumulator per graph, shared across a whole test session (a
+    session-scoped pytest fixture, in practice) -- coverage is a property of
+    the *suite*, not of any single test.
+    """
+
+    topology: GraphTopology
+    _visited: set[str] = field(default_factory=set)
+    _taken: set[Branch] = field(default_factory=set)
+    runs_recorded: int = 0
+
+    def record(self, trace: RunTrace) -> None:
+        self._visited.update(trace.nodes_visited)
+        self._taken.update(trace.edges_taken & self.topology.branches)
+        self.runs_recorded += 1
+
+    def record_run(self, compiled_graph: _StreamableGraph, input: Any) -> RunTrace:
+        """Run the graph and record the trace in one step."""
+        trace = trace_run(compiled_graph, input)
+        self.record(trace)
+        return trace
+
+    def coverage(self) -> BranchCoverage:
+        return BranchCoverage(
+            topology=self.topology,
+            nodes_visited=frozenset(self._visited) & self.topology.nodes,
+            branches_taken=frozenset(self._taken),
+        )