llm-ports 0.12.1.dev14__py3-none-any.whl

llm_ports/__init__.py

@@ -0,0 +1,88 @@
+"""Thin Protocols for LLM work, and the one call path around them.
+
+The package root exports:
+
+- the ports and their value types (`ports`);
+- the call path `call.generate`, which resolves a `Role` from the consumer's tier table (`roles`),
+  renders a prompt pack named by its content sha (`packs`), runs per-call graders whose failures
+  are recorded as scores (`call_graders`), and ends in a named `Outcome` (`ok`, `gated`,
+  `unavailable`, `request_fault`, `truncated` or `invalid`);
+- the data-rights gate (`content.print_gate`) and `GenerationFailed`.
+
+Frameworks live behind the ports. `pydantic_ai_generator.PydanticAIGenerator` implements
+`Generator` and `pydantic_ai_embedder.PydanticAIEmbedder` implements `Embedder`; import adapters
+by module path.
+
+Invariants:
+    - The package root imports no framework.
+    - A `Generator` never raises for a model, wire or gate failure.
+
+Rationale: ADR-0001.
+"""
+
+from llm_ports.call import Hooks, generate, json_output, request_for, require_ok, tracer_trace
+from llm_ports.content import Activity, ContentItem, DataClass, GateVerdict, print_gate
+from llm_ports.failure import GenerationFailed
+from llm_ports.ports import (
+    Chunk,
+    CitedAnswer,
+    Claim,
+    Embedded,
+    Embedder,
+    EmbedInput,
+    Generated,
+    Generation,
+    Generator,
+    GenRequest,
+    Judge,
+    JudgeVerdict,
+    Outcome,
+    Prompt,
+    PromptSource,
+    Reranked,
+    Reranker,
+    Seed,
+    Tracer,
+)
+from llm_ports.roles import Role, Tier, TierTable, load_tiers
+
+__version__ = "0.12.0"
+
+__all__ = [
+    "Activity",
+    "Chunk",
+    "CitedAnswer",
+    "Claim",
+    "ContentItem",
+    "DataClass",
+    "GateVerdict",
+    "EmbedInput",
+    "Embedded",
+    "Embedder",
+    "GenRequest",
+    "Generated",
+    "Generation",
+    "GenerationFailed",
+    "Generator",
+    "Hooks",
+    "Judge",
+    "JudgeVerdict",
+    "Outcome",
+    "Prompt",
+    "PromptSource",
+    "Reranked",
+    "Reranker",
+    "Role",
+    "Seed",
+    "Tier",
+    "TierTable",
+    "Tracer",
+    "__version__",
+    "generate",
+    "json_output",
+    "load_tiers",
+    "print_gate",
+    "request_for",
+    "require_ok",
+    "tracer_trace",
+]

llm_ports/call.py

@@ -0,0 +1,301 @@
+"""The single generation path: role, request, gate, trace, `Generator`, graders.
+
+Example:
+    res = await call.generate(CATEGORIZE, prompt=text, output=Reply, generator=gen, hooks=HOOKS)
+    if res.outcome != "ok": ...  # gated | unavailable | request_fault | truncated | invalid
+
+Per call, the role resolves to one `GenRequest` (`request_for`), the category is entered with
+`context.llm_feature`, and the `admit` hook runs. A refusal returns `gated` before any trace opens:
+a switched-off category is not an error. Otherwise one trace observation (the `trace` hook) wraps
+the `Generator` call. Every `ok` reply is scored by the graders its role names; each score is
+written on the observation and carried under `stamp["scores"]`, then handed to `on_scores` (a
+gauge) and `on_ok` (a sampler). Feature code carries no model literal, no timeout and no framework
+import: the role carries them, and the `Generator` is passed in (an adapter, a test stub, a
+recording replay).
+
+The hooks are where a consumer's own planes plug in: a category switchboard, a tracing backend, a
+metrics sink, an LLM-judge sampler. Every hook is optional. With none, the path is role,
+`Generator`, graders, and the outcome is still named and a grader failure still recorded as a score.
+
+Invariants:
+    - `generate` never raises for a model, wire or gate failure; the outcome names it.
+    - A non-`ok`, non-`gated` outcome ends the observation in error and emits `llm_call_failed`.
+      `_NotOk` is raised inside the trace block only and never escapes.
+    - A caller whose ledger is an `except` uses `require_ok`.
+
+Rationale: ADR-0001.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from contextlib import AbstractAsyncContextManager
+from dataclasses import dataclass, replace
+from typing import Any
+
+from llm_ports import ports
+from llm_ports.call_graders import CallGrader, CallScore, graders_for_inputs, score_all
+from llm_ports.context import llm_feature
+from llm_ports.failure import GenerationFailed
+from llm_ports.observation import Observation, observe
+from llm_ports.ports import Generated, Generator, GenRequest, Tracer
+from llm_ports.reference import NoopTracer
+from llm_ports.roles import Role
+
+OUTPUT_TRACE_MAX = 4000
+_log = logging.getLogger("llm_ports.call")
+
+
+#: Trace hook: `(role, request, prompt, metadata)` to an async context manager yielding an
+#: `Observation`. On an exception inside the block, the hook ends the observation in error and lets
+#: the exception propagate.
+TraceHook = Callable[[Role, GenRequest, str, Mapping[str, Any]], AbstractAsyncContextManager[Observation]]
+#: Admission hook: `(role)` to None to admit, or to the refusal reason (the outcome becomes `gated`).
+AdmitHook = Callable[[Role], str | None]
+#: Event hook: `(logging level, event, fields)`. Without one, events go to the stdlib logger
+#: `llm_ports.call`.
+EventHook = Callable[[int, str, Mapping[str, Any]], None]
+
+
+@dataclass(frozen=True)
+class Hooks:
+    """Optional consumer hooks for `generate`.
+
+    Attributes:
+        admit: Runs before any trace opens; a returned reason makes the outcome `gated`.
+        trace: Opens the call's observation; without one, a sink-less tracer is used.
+        graders: The grader table by name; None means `call_graders.GRADERS`.
+        on_scores: Awaited with the role and its scores whenever a reply was graded.
+        on_ok: Called with the role, prompt and result after an `ok` reply.
+        on_event: Receives `llm_call_failed` and `llm_call_graded`; without one, they are logged.
+    """
+
+    admit: AdmitHook | None = None
+    trace: TraceHook | None = None
+    graders: Mapping[str, CallGrader] | None = None  # None means call_graders.GRADERS
+    on_scores: Callable[[Role, tuple[CallScore, ...]], Awaitable[Any]] | None = None
+    on_ok: Callable[[Role, str, Generated[Any]], Any] | None = None
+    on_event: EventHook | None = None
+
+
+def tracer_trace(tracer: Tracer, *, provider: str = "openai") -> TraceHook:
+    """Return a `trace` hook over any `Tracer` port adapter (Langfuse, OTel, a test recorder).
+
+    The hook opens one generation per call (`observation.observe`), named by the role's category,
+    and ends it in error on a non-`ok` outcome.
+    """
+
+    def trace(
+        role: Role, req: GenRequest, prompt: str, meta: Mapping[str, Any]
+    ) -> AbstractAsyncContextManager[Observation]:
+        return observe(
+            tracer,
+            name=role.category,
+            model=req.model,
+            provider=provider,
+            input=prompt,
+            metadata=meta,
+            prompt_name=role.pack,
+        )
+
+    return trace
+
+
+#: The trace used when no hook is given: the port path over the sink-less tracer, so the package
+#: has one null sink (`reference.NoopGeneration`) at either layer.
+_null_trace = tracer_trace(NoopTracer())
+
+
+class _NotOk(Exception):
+    """Ends the observation in error; raised inside the trace block only and never escapes."""
+
+
+def request_for(
+    role: Role,
+    *,
+    prompt: str,
+    system: str = "",
+    images: Sequence[tuple[bytes, str]] = (),
+    settings: Mapping[str, Any] | None = None,
+) -> GenRequest:
+    """Resolve the role into one wire request.
+
+    Pure. `settings` override the role's model settings for this call (a seed for a vote, a tighter
+    `max_tokens`); the role stays the default.
+    """
+    tier = role.resolved_tier
+    return GenRequest(
+        model=role.resolved_model,
+        prompt=prompt,
+        system=system,
+        images=tuple(images),
+        settings={**role.model_settings(), **(settings or {})},
+        box=tier.box,
+        timeout_s=role.resolved_timeout_s,
+        output_retries=role.output_retries,
+        label=role.name,
+    )
+
+
+def json_output(schema: Mapping[str, Any], name: str) -> type[dict[str, Any]]:
+    """Return a hand-written JSON schema as an `output=` type.
+
+    The schema goes to the model unchanged as the grammar, and the value is a plain dict. `name` is
+    what a test stub dispatches on. The result is a port value type (`ports.JsonOutput`); the
+    `Generator` adapter translates it.
+    """
+    return ports.json_output(schema, name)
+
+
+def require_ok[T](res: Generated[T]) -> T:
+    """Return the result's value, or raise its failure.
+
+    For a caller whose `except Exception` is its ledger: under the never-raise contract a dead model
+    would otherwise become an unrecorded None.
+
+    Raises:
+        GenerationFailed: `(outcome, reason)` for any non-`ok` outcome, `gated` included, and for an
+            `ok` result without a value.
+    """
+    if res.outcome != "ok" or res.value is None:
+        raise GenerationFailed(res.outcome, res.reason)
+    return res.value
+
+
+def trace_output(value: Any) -> str:
+    """Return the observation's output text, capped at `OUTPUT_TRACE_MAX` characters.
+
+    A pydantic model or a mapping is rendered as JSON; None is the empty string.
+    """
+    if value is None:
+        return ""
+    dump = getattr(value, "model_dump_json", None)
+    if callable(dump):
+        return str(dump())[:OUTPUT_TRACE_MAX]
+    if isinstance(value, Mapping):
+        return json.dumps(value, default=str)[:OUTPUT_TRACE_MAX]
+    return str(value)[:OUTPUT_TRACE_MAX]
+
+
+def _event(h: Hooks, level: int, event: str, **fields: Any) -> None:
+    if h.on_event is not None:
+        h.on_event(level, event, fields)
+    else:
+        _log.log(level, "%s %s", event, fields)
+
+
+def _grade(role: Role, prompt: str, res: Generated[Any], obs: Observation, h: Hooks) -> Generated[Any]:
+    """Score an `ok` reply with the role's graders and write each score on its observation.
+
+    The scores ride in the returned stamp under `"scores"`. Failed graders are also emitted by name
+    as `llm_call_graded`.
+    """
+    scores = score_all(role, prompt, res.value, h.graders)
+    for s in scores:
+        obs.score(name=f"{role.name}.{s.name}", value=s.value, comment=s.reason or None)
+    failed = [s.name for s in scores if not s.passed]
+    if failed:
+        _event(
+            h,
+            logging.INFO,
+            "llm_call_graded",
+            role=role.name,
+            failed=failed,
+            reasons={s.name: s.reason for s in scores if not s.passed},
+        )
+    return replace(res, stamp={**res.stamp, "scores": scores})
+
+
+async def generate[T](
+    role: Role,
+    *,
+    prompt: str,
+    output: type[T] = str,  # type: ignore[assignment]
+    generator: Generator,
+    hooks: Hooks | None = None,
+    system: str = "",
+    images: Sequence[tuple[bytes, str]] = (),
+    validate: Callable[[T], Sequence[Any]] | None = None,
+    retry_message: Callable[[Sequence[Any]], str] | None = None,
+    settings: Mapping[str, Any] | None = None,
+    metadata: Mapping[str, Any] | None = None,
+    web: bool = False,
+) -> Generated[T]:
+    """Run one generation for `role` through `generator`.
+
+    The path and the hooks are described in the module docstring. `web=True` says the role's tier
+    fetches pages the prompt never rendered: like an image, an input no text grader sees, so
+    `call_graders.TEXT_ONLY_GRADERS` are dropped for this call. The role itself is unchanged.
+
+    Returns:
+        The result, with a named outcome. A model, wire or gate failure is an outcome, never an
+        exception.
+
+    Raises:
+        RuntimeError: Only when a trace hook swallowed a generator exception, leaving no result.
+    """
+    h = hooks or Hooks()
+    seen = graders_for_inputs(tuple(role.graders), has_images=bool(images), has_web=web)
+    if seen != tuple(role.graders):
+        role = replace(role, graders=seen)
+    req = request_for(role, prompt=prompt, system=system, images=images, settings=settings)
+    with llm_feature(role.category):
+        if h.admit is not None:
+            refused = h.admit(role)
+            if refused is not None:
+                return Generated(value=None, outcome="gated", model=req.model, reason=refused[:200])
+        res: Generated[T] | None = None
+        trace = h.trace or _null_trace
+        try:
+            async with trace(role, req, prompt, {"role": role.name, **(metadata or {})}) as obs:
+                res = await generator.generate(req, output, validate=validate, retry_message=retry_message)
+                obs.update(
+                    output=trace_output(res.value),
+                    usage_details=dict(res.usage or {}),
+                    metadata={
+                        "role": role.name,
+                        "outcome": res.outcome,
+                        "retries": res.retries,
+                        "finish_reason": res.finish_reason,
+                        "problems": len(res.problems),
+                    },
+                )
+                res = replace(res, stamp={**res.stamp, "generation": obs})
+                if res.outcome == "ok" and role.graders:
+                    res = _grade(role, prompt, res, obs, h)
+                if res.outcome not in ("ok", "gated"):
+                    raise _NotOk(f"{res.outcome}: {res.reason}")
+        except _NotOk:
+            _event(
+                h,
+                logging.WARNING,
+                "llm_call_failed",
+                role=role.name,
+                outcome=res.outcome if res else None,
+                reason=res.reason if res else None,
+            )
+        if res is None:  # reachable only when a trace hook swallowed a generator exception
+            raise RuntimeError(f"llm_call {role.name}: the generator returned nothing")
+        if res.stamp.get("scores") and h.on_scores is not None:
+            await h.on_scores(role, res.stamp["scores"])
+        if res.outcome == "ok" and h.on_ok is not None:
+            h.on_ok(role, prompt, res)
+        return res
+
+
+__all__ = [
+    "OUTPUT_TRACE_MAX",
+    "AdmitHook",
+    "EventHook",
+    "Hooks",
+    "Observation",
+    "TraceHook",
+    "generate",
+    "json_output",
+    "request_for",
+    "require_ok",
+    "trace_output",
+    "tracer_trace",
+]

llm_ports/call_graders.py

@@ -0,0 +1,297 @@
+"""Deterministic graders over one generation: the prompt in, the value out.
+
+Pure. A `CallGrader` is `(prompt, value, role) -> CallScore`: a numeric value (what a trace score
+and a dashboard can sum), a pass or fail against the grader's own bar, and a reason that names the
+violated property on failure. `call.generate` runs the graders a `Role.graders` names on every
+`ok` reply and writes each as a score on the call's trace observation. A golden eval runs the same
+functions over a pack's golden set (`board_graders`), so the live score and the offline verdict
+agree on what "complete" or "echoed" means.
+
+Built-in graders, for structured outputs (a dict or pydantic value under a grammar):
+
+- `schema_complete`: the share of fields that carry a value. A strict grammar admits `""` for
+  every required key, so a parsed reply is not necessarily an answered one. An empty list is an
+  answer; only None and blank text count as unanswered. On a pydantic value, None on a field whose
+  annotation admits None (`float | None`, `Optional[X]`) is an answer too, while None on a required
+  field fails. A dict value has no annotations to read, so every None in it is unanswered.
+- `numbers_from_inputs`: figures in the reply's prose that no part of the prompt rendered, digit
+  runs by value and spelled numbers alike (`numbers.loose`): the number law, per call.
+- `echo`: the share of the reply's sentences copied verbatim from the prompt.
+
+A consumer adds its own graders by passing a table (`{**GRADERS, "mine": fn}`) to `score_all`,
+`board_graders` or `call.Hooks(graders=...)`. This package never mutates `GRADERS`.
+
+Invariants:
+    - `score_one` and `score_all` never raise. A grader's exception becomes a failed score of value
+      `CRASHED` (-1, below every honest value, all of which are >= 0) whose reason names the
+      exception, so a crashed grader never reads as a pass.
+
+Rationale: ADR-0003.
+"""
+
+from __future__ import annotations
+
+import re
+import types
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import Annotated, Any, Union, get_args, get_origin
+
+from pydantic import BaseModel
+
+from llm_ports.numbers import digits, loose
+from llm_ports.roles import Role
+
+ECHO_MAX_SHARE = 0.5  # `echo` fails at or above this share of copied sentences
+ECHO_MIN_CHARS = 40  # a sentence shorter than this is a label ("unclear", a key), not prose
+CRASHED = -1.0  # the value of a grader that raised; every honest value is >= 0
+_SENTENCE = re.compile(r"[.!?\n]+")
+_WS = re.compile(r"\s+")
+_INDEX = re.compile(r"\[\d+\]")  # `items[3].price` -> `items[].price`: the field, not the row
+
+
+@dataclass(frozen=True)
+class CallScore:
+    """One grader's verdict on one reply.
+
+    Attributes:
+        name: The grader's name.
+        value: The numeric score; `CRASHED` when the grader raised.
+        passed: Whether the value meets the grader's own bar.
+        reason: Empty on a pass; names the violated property on a failure.
+    """
+
+    name: str
+    value: float
+    passed: bool
+    reason: str = ""
+
+
+CallGrader = Callable[[str, Any, Role], CallScore]
+
+
+def leaves(value: Any, path: str = "") -> Iterator[tuple[str, Any]]:
+    """Yield `(path, scalar)` for every leaf of a dict, pydantic or list value, in order.
+
+    Pure. An empty list is itself a leaf.
+    """
+    dump = getattr(value, "model_dump", None)
+    if callable(dump):
+        value = dump()
+    if isinstance(value, Mapping):
+        for k, v in value.items():
+            yield from leaves(v, f"{path}.{k}" if path else str(k))
+    elif isinstance(value, (list, tuple)):
+        if not value:
+            yield path, value  # an empty list is a leaf: the field answered nothing
+        for i, v in enumerate(value):
+            yield from leaves(v, f"{path}[{i}]")
+    else:
+        yield path, value
+
+
+def prose(value: Any) -> list[str]:
+    """Return every string leaf of the reply: what the number law and the echo check read."""
+    return [v for _p, v in leaves(value) if isinstance(v, str)]
+
+
+def _empty(v: Any) -> bool:
+    return v is None or (isinstance(v, str) and not v.strip())
+
+
+def _admits_none(ann: Any) -> bool:
+    """Return whether the annotation accepts None (`X | None`, `Optional[X]`, `Annotated[X | None, ...]`)."""
+    if ann is None or ann is type(None):
+        return True
+    origin = get_origin(ann)
+    if origin is Annotated:
+        return _admits_none(get_args(ann)[0])
+    return origin in (Union, types.UnionType) and any(_admits_none(a) for a in get_args(ann))
+
+
+def _models_in(ann: Any, suffix: str = "") -> Iterator[tuple[str, type[BaseModel]]]:
+    """Yield `(path suffix, model)` for every `BaseModel` an annotation reaches.
+
+    The suffix is `""` for the model itself or through a union, plus `[]` per list, tuple or set
+    layer: the spelling `leaves` emits, without indexes.
+    """
+    if isinstance(ann, type) and issubclass(ann, BaseModel):
+        yield suffix, ann
+        return
+    origin, args = get_origin(ann), get_args(ann)
+    if origin is Annotated or origin in (Union, types.UnionType):
+        for a in args[:1] if origin is Annotated else args:
+            yield from _models_in(a, suffix)
+    elif isinstance(origin, type) and issubclass(origin, (list, tuple, set, frozenset)):
+        for a in args:
+            if a is not Ellipsis:
+                yield from _models_in(a, suffix + "[]")
+
+
+def nullable_paths(model: type[BaseModel], prefix: str = "", _seen: frozenset[type] = frozenset()) -> frozenset[str]:
+    """Return the index-free leaf paths of every field whose annotation admits None.
+
+    Pure. Paths (`items[].price`) run through nested models and lists of models. They are read off
+    the class, so they never depend on which rows a reply carried. A self-referential model stops at
+    its own repeat.
+    """
+    out: set[str] = set()
+    seen = _seen | {model}
+    for name, field in model.model_fields.items():
+        path = f"{prefix}.{name}" if prefix else name
+        if _admits_none(field.annotation):
+            out.add(path)
+        for suffix, sub in _models_in(field.annotation):
+            if sub not in seen:
+                out |= nullable_paths(sub, path + suffix, seen)
+    return frozenset(out)
+
+
+def schema_complete(_prompt: str, value: Any, _role: Role) -> CallScore:
+    """Score the share of the reply's fields that carry a value.
+
+    Passes only when no field is unanswered. None and blank text are unanswered; an empty list is an
+    answer, and so is None on a pydantic field whose annotation admits None.
+    """
+    fields = list(leaves(value))
+    if not fields:
+        return CallScore("schema_complete", 0.0, False, "the reply carries no fields")
+    nullable = nullable_paths(type(value)) if isinstance(value, BaseModel) else frozenset()
+    empty = [p for p, v in fields if _empty(v) and _INDEX.sub("[]", p) not in nullable]
+    share = round(1 - len(empty) / len(fields), 4)
+    return CallScore("schema_complete", share, not empty, f"empty: {empty[:4]}" if empty else "")
+
+
+def numbers_from_inputs(prompt: str, value: Any, _role: Role) -> CallScore:
+    """Count the distinct figures in the reply's prose that the prompt did not render.
+
+    Digit runs compare by value, and spelled numbers count too (`numbers.loose`). The prompt's own
+    text licenses a spelled word, and '%' licenses 'percent'. Passes only at zero.
+
+    Rationale: ADR-0003.
+    """
+    have = digits(prompt)
+    hits = sorted({n for s in prose(value) for n in loose(s, have, prompt)})
+    return CallScore(
+        "numbers_from_inputs", float(len(hits)), not hits, f"numbers no input rendered: {hits[:3]}" if hits else ""
+    )
+
+
+def _sentences(text: str) -> list[str]:
+    return [s for s in (_WS.sub(" ", p).strip().lower() for p in _SENTENCE.split(text)) if len(s) >= ECHO_MIN_CHARS]
+
+
+def echo(prompt: str, value: Any, _role: Role) -> CallScore:
+    """Score the share of the reply's sentences copied verbatim from the prompt.
+
+    The compare ignores case and whitespace, and a sentence shorter than `ECHO_MIN_CHARS` is not
+    counted; a reply with no counted sentence passes at 0. Fails at or above `ECHO_MAX_SHARE`.
+    """
+    hay = _WS.sub(" ", prompt).lower()
+    sents = [s for p in prose(value) for s in _sentences(p)]
+    if not sents:
+        return CallScore("echo", 0.0, True)
+    copied = [s for s in sents if s in hay]
+    share = round(len(copied) / len(sents), 4)
+    return CallScore(
+        "echo",
+        share,
+        share < ECHO_MAX_SHARE,
+        f"{len(copied)}/{len(sents)} sentences copied from the prompt: {copied[0][:60]!r}"
+        if share >= ECHO_MAX_SHARE
+        else "",
+    )
+
+
+#: The built-in graders by name; a `Role` names its graders as strings (data, comparable, printable).
+GRADERS: dict[str, CallGrader] = {
+    "schema_complete": schema_complete,
+    "numbers_from_inputs": numbers_from_inputs,
+    "echo": echo,
+}
+
+
+#: Graders whose verdict holds only when the prompt text is the call's whole input. A grader sees
+#: `(prompt, value, role)`, so an image sent beside the prompt is invisible to it, as are the pages
+#: a web-fetching tier read. `numbers_from_inputs` fails every honest reply on such a call: a chart
+#: read's figures come off the image, a research reply's off the pages. `call.generate` drops these
+#: for any call with `images=` or `web=True`; a text-only call keeps every grader its role names.
+TEXT_ONLY_GRADERS: frozenset[str] = frozenset({"numbers_from_inputs"})
+
+
+def graders_for_inputs(graders: tuple[str, ...], *, has_images: bool, has_web: bool = False) -> tuple[str, ...]:
+    """Return the graders that can see this call's inputs, in their given order.
+
+    Pure. A text-only call keeps all of them; an image beside the prompt or a web-fetching tier
+    removes `TEXT_ONLY_GRADERS`.
+    """
+    if has_images or has_web:
+        return tuple(g for g in graders if g not in TEXT_ONLY_GRADERS)
+    return graders
+
+
+def score_one(
+    name: str, prompt: str, value: Any, role: Role, table: Mapping[str, CallGrader] | None = None
+) -> CallScore:
+    """Run one grader by name from `table` (default `GRADERS`).
+
+    Never raises: an unknown name or a grader exception becomes a failed score of value `CRASHED`.
+    """
+    fn = (GRADERS if table is None else table).get(name)
+    if fn is None:
+        return CallScore(name, CRASHED, False, f"no grader named {name!r}")
+    try:
+        return fn(prompt, value, role)
+    except Exception as exc:  # a crashed grader must not read as a pass
+        return CallScore(name, CRASHED, False, f"grader raised {type(exc).__name__}: {exc}"[:200])
+
+
+def score_all(
+    role: Role, prompt: str, value: Any, table: Mapping[str, CallGrader] | None = None
+) -> tuple[CallScore, ...]:
+    """Run every grader the role names, in the role's order; never raises."""
+    return tuple(score_one(n, prompt, value, role, table) for n in role.graders)
+
+
+def board_graders(
+    role: Role,
+    table: Mapping[str, CallGrader] | None = None,
+) -> dict[str, Callable[[dict[str, Any]], tuple[bool, str]]]:
+    """Return the same graders in the `Evaluator` row shape (`ports.Grader`).
+
+    A golden row carries `prompt` and `value`. A row whose model never answered (`value is None`)
+    fails every grader with its `reason`, so the board names the outage rather than grading an
+    absence.
+    """
+
+    def bind(name: str) -> Callable[[dict[str, Any]], tuple[bool, str]]:
+        def grader(row: dict[str, Any]) -> tuple[bool, str]:
+            if row.get("value") is None:
+                return False, f"no reply: {row.get('reason') or row.get('outcome')}"
+            s = score_one(name, str(row.get("prompt") or ""), row["value"], role, table)
+            return s.passed, s.reason
+
+        return grader
+
+    return {n: bind(n) for n in role.graders}
+
+
+__all__ = [
+    "CRASHED",
+    "ECHO_MAX_SHARE",
+    "ECHO_MIN_CHARS",
+    "GRADERS",
+    "TEXT_ONLY_GRADERS",
+    "CallGrader",
+    "CallScore",
+    "board_graders",
+    "echo",
+    "graders_for_inputs",
+    "leaves",
+    "nullable_paths",
+    "numbers_from_inputs",
+    "prose",
+    "schema_complete",
+    "score_all",
+    "score_one",
+]