archforge-optimizer 0.1.0__py3-none-any.whl

archforge/judge/base.py

@@ -0,0 +1,257 @@
+"""The Judge — LLM-as-judge scoring of a run (spec §3, §4, §5).
+
+`Judge.score(trace, task, rubric_id) -> RunScore` is ONE LLM call that turns a
+trace into a scored verdict: an aggregate, named rubric dimensions, a
+confidence, and a *per-step breakdown* (StepScore[]) naming which agent lost
+which points. The per-step breakdown is the credit-assignment raw material the
+Architect consumes — but the Judge contains NO diagnosis or proposal
+text; per the locked "keep split" decision it scores only.
+
+`aggregate_scores(...)` turns a list of RunScores (R repeats × N tasks) into a
+single `SuiteAggregate` (confidence-weighted mean + per-task means). It is pure
+math — no LLM call — so the call budget stays at "1 judge call per scored run".
+rubric_id is stamped on every RunScore and the aggregate (invariant I5, E2).
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, Sequence, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from archforge import userconfig as ucfg
+import archforge.models as m
+from archforge.host.base import Task
+from archforge.llm.base import LLMClient, LLMError, Message, Role
+
+
+# --------------------------------------------------------------------------- #
+# Rubric — the scoring schema, versioned by rubric_id (E2 / I5)
+# --------------------------------------------------------------------------- #
+
+
+class Rubric(BaseModel):
+    """A named, versioned scoring schema: sub-rubric -> human description.
+
+    `rubric_id` is the version key recorded on every RunScore, so the Gatekeeper
+    never compares scores graded under different rubrics (invariant I5).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    rubric_id: str
+    sub_rubrics: dict[str, str]  # dimension name -> "what a high score looks like"
+
+
+def default_rubric() -> Rubric:
+    """The Judge's default rubric (E2 / I5), built lazily from the active config.
+
+    A function (not a module-level instance) so the rubric id + sub-rubrics resolve
+    from ``archforge.userconfig`` at first USE — not at import — which keeps this
+    module importable before ``init`` has run and makes an edit to
+    ``.archforge/archforge.py`` take effect on a fresh construction. Each call reads
+    the cached resolved namespace (one resolve per process) and builds a fresh
+    ``Rubric``; cheap and free of cross-test stale-state.
+    """
+    return Rubric(
+        rubric_id=ucfg.get("DEFAULT_RUBRIC_ID"),
+        sub_rubrics=dict(ucfg.get("DEFAULT_SUB_RUBRICS")),
+    )
+
+
+# --------------------------------------------------------------------------- #
+# SuiteAggregate — a suite's scores rolled into one comparable number
+# --------------------------------------------------------------------------- #
+
+
+class SuiteAggregate(BaseModel):
+    """Confidence-weighted aggregation of a suite's RunScores (spec §5).
+
+    `mean` is what the Gatekeeper compares across incumbent vs candidate (same
+    rubric_id + same suite, per I5). `per_task` supports per-task diagnosis and
+    suite-aware plateau reasoning.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    suite_id: str
+    rubric_id: str
+    mean: float
+    n_runs: int
+    per_task: dict[str, float] = Field(default_factory=dict)
+    confidence: float = 1.0
+    # Set by the SuiteRunner when too many tasks crashed (spec E4).
+    unrunnable: bool = False
+
+
+# --------------------------------------------------------------------------- #
+# Judge protocol — real Judge and ScriptedJudge both satisfy this
+# --------------------------------------------------------------------------- #
+
+
+@runtime_checkable
+class JudgeProtocol(Protocol):
+    """What the SuiteRunner depends on: score one run, aggregate a suite."""
+
+    def score(self, trace: m.Trace, task: Task, rubric_id: str) -> m.RunScore: ...
+
+    def score_suite(
+        self, scores: Sequence[m.RunScore], *, suite_id: str, rubric_id: str
+    ) -> SuiteAggregate: ...
+
+
+# --------------------------------------------------------------------------- #
+# The real Judge — one LLM call per trace, JSON-structured scoring
+# --------------------------------------------------------------------------- #
+
+
+class Judge:
+    """Scores runs via an LLM client. Provider-agnostic — depends on `LLMClient` only.
+
+    On any failure (provider error or malformed JSON judgment) `score` raises
+    `LLMError`; the caller (SuiteRunner) does bounded retry with backoff (E9)
+    rather than fabricating a score.
+    """
+
+    def __init__(
+        self,
+        llm: LLMClient,
+        *,
+        model: str,
+        rubric: Rubric | None = None,
+    ) -> None:
+        self._llm = llm
+        self._model = model
+        self._rubric = rubric if rubric is not None else default_rubric()
+
+    @property
+    def rubric(self) -> Rubric:
+        return self._rubric
+
+    def score(self, trace: m.Trace, task: Task, rubric_id: str) -> m.RunScore:
+        messages = self._build_messages(trace, task, rubric_id)
+        completion = self._llm.complete(
+            messages, model=self._model, temperature=0.0, response_format="json",
+        )
+        parsed = completion.parsed
+        if not isinstance(parsed, dict):
+            raise LLMError("judge returned no parsed JSON judgment")
+        return self._to_run_score(parsed, trace, rubric_id)
+
+    def score_suite(
+        self, scores: Sequence[m.RunScore], *, suite_id: str, rubric_id: str
+    ) -> SuiteAggregate:
+        return aggregate_scores(scores, suite_id=suite_id, rubric_id=rubric_id)
+
+    # --------------------------------------------------------------- internals
+    def _build_messages(
+        self, trace: m.Trace, task: Task, rubric_id: str
+    ) -> list[Message]:
+        return [
+            Message(
+                role=Role.SYSTEM,
+                content=(
+                    "You are a strict LLM-as-judge. Score a multi-agent run against "
+                    f"rubric `{rubric_id}`. Each score is a float in [0, 1]. Return "
+                    "ONLY JSON with: `aggregate` (overall 0-1), `confidence` (0-1), "
+                    "`rubric_scores` (per dimension 0-1), and `step_scores` (one per "
+                    "step: {node_id, sub_rubrics: {dim: 0-1}, note})."
+                ),
+            ),
+            Message(
+                role=Role.USER,
+                content=(
+                    f"TASK INPUT:\n{task.input}\n\n"
+                    f"RUBRIC ({rubric_id}):\n"
+                    + "\n".join(f"- {k}: {v}" for k, v in self._rubric.sub_rubrics.items())
+                    + "\n\nRUN TRACE (steps in order):\n"
+                    + _trace_summary(trace)
+                    + "\n\nReturn the JSON judgment."
+                ),
+            ),
+        ]
+
+    def _to_run_score(
+        self, parsed: dict, trace: m.Trace, rubric_id: str
+    ) -> m.RunScore:
+        try:
+            aggregate = float(parsed["aggregate"])
+            confidence = float(parsed.get("confidence", 1.0))
+            rubric_scores = {k: float(v) for k, v in parsed.get("rubric_scores", {}).items()}
+            step_scores = [
+                m.StepScore(
+                    node_id=str(s["node_id"]),
+                    sub_rubrics={k: float(v) for k, v in s.get("sub_rubrics", {}).items()},
+                    note=s.get("note"),
+                )
+                for s in parsed.get("step_scores", [])
+            ]
+        except (KeyError, TypeError, ValueError) as exc:
+            raise LLMError(f"malformed judge JSON: {exc}") from exc
+
+        return m.RunScore(
+            run_id=trace.run_id,
+            spec_id=trace.spec_id,
+            task_id=trace.task_id,
+            rubric_scores=rubric_scores,
+            aggregate=aggregate,
+            confidence=confidence,
+            judge_meta=m.JudgeMeta(model=self._model, rubric_id=rubric_id),
+            step_scores=step_scores,
+        )
+
+
+# --------------------------------------------------------------------------- #
+# Pure aggregation — shared by the real Judge and ScriptedJudge (1 impl)
+# --------------------------------------------------------------------------- #
+
+
+def aggregate_scores(
+    scores: Sequence[m.RunScore], *, suite_id: str, rubric_id: str
+) -> SuiteAggregate:
+    """Confidence-weighted mean of a suite's RunScores (pure, no LLM).
+
+    Equal weights when total confidence is zero (degenerate but well-defined).
+    Per-task means are simple averages — the per-step breakdown is retained on
+    each RunScore for the Architect; the aggregate only needs task-level means.
+    """
+
+    scores = list(scores)
+    n = len(scores)
+    if n == 0:
+        return SuiteAggregate(suite_id=suite_id, rubric_id=rubric_id, mean=0.0, n_runs=0,
+                              per_task={}, confidence=1.0)
+
+    total_w = sum(s.confidence for s in scores)
+    if total_w <= 0:
+        mean = sum(s.aggregate for s in scores) / n
+    else:
+        mean = sum(s.aggregate * s.confidence for s in scores) / total_w
+
+    per_task: dict[str, list[float]] = {}
+    for s in scores:
+        per_task.setdefault(s.task_id, []).append(s.aggregate)
+    per_task_mean = {tid: sum(vs) / len(vs) for tid, vs in per_task.items()}
+
+    return SuiteAggregate(
+        suite_id=suite_id, rubric_id=rubric_id, mean=mean, n_runs=n,
+        per_task=per_task_mean, confidence=total_w / n,
+    )
+
+
+def _trace_summary(trace: m.Trace) -> str:
+    if not trace.steps:
+        return "(no steps recorded — empty run)"
+    lines: list[str] = []
+    for i, step in enumerate(trace.steps):
+        lines.append(
+            f"[{i}] node={step.node_id}\n    prompt_in: {step.prompt_in}\n"
+            f"    response_out: {step.response_out}"
+        )
+    return "\n".join(lines)
+
+
+__all__ = [
+    "Rubric", "default_rubric", "SuiteAggregate", "JudgeProtocol",
+    "Judge", "aggregate_scores",
+]

archforge/judge/scripted.py

@@ -0,0 +1,145 @@
+"""ScriptedJudge — a deterministic fake `JudgeProtocol`.
+
+Used by every scenario test that needs a Judge without a real model. It honors
+the same contract as the real Judge and reuses the pure `aggregate_scores`, so
+aggregation behaviour is identical real-vs-fake.
+
+Scripting hooks (the things scenario tests need):
+  * `set_aggregate(...)` / fixed per-(spec_id, task_id) tables -> deterministic scores
+  * `with_noise_band(width)` -> jitter the aggregate within ±width of a base (E1)
+  * `regress_after(spec_id, drop)` -> score the next-scoring spec lower by `drop`
+    (used to prove promoted-then-regresses rollback, E6)
+  * `raise_on_next(...)` -> force the next score() call to raise LLMError (E9)
+
+It records every run it scores, including the per-step breakdown, so the
+Architect and tests can assert on credit-assignment input. If a run
+is unknown and no base/scripted behaviour covers it, it returns a neutral 0.5
+(never silently wrong — tests configure exactly what they need).
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+from archforge.config import SCRIPTED_NOISE_PATTERN
+import archforge.models as m
+from archforge.host.base import Task
+from archforge.judge.base import JudgeProtocol, Rubric, SuiteAggregate, aggregate_scores, default_rubric
+from archforge.llm.base import LLMError
+
+
+class ScriptedJudge:
+    """A programmable, deterministic Judge (no LLM)."""
+
+    def __init__(self, *, rubric: Rubric | None = None) -> None:
+        self._rubric = rubric if rubric is not None else default_rubric()
+        # (spec_id, task_id) -> aggregate
+        self._fixed: dict[tuple[str, str], float] = {}
+        # base aggregate when not explicitly fixed
+        self._base: float = 0.5
+        # noise band applied to the aggregate (spec E1), deterministic per run order
+        self._noise_width: float = 0.0
+        # forced regression: after seeing `trigger_spec_id` once, subtract `drop`
+        self._regress: tuple[str, float] | None = None
+        self._regress_armed: bool = False
+        # raise on the next score() call (spec E9 grader outage)
+        self._raise_next: Exception | None = None
+        # recorded verdicts for assertions
+        self.scored: list[m.RunScore] = []
+
+    # --------------------------------------------------------------- config
+    def using(self, rubric: Rubric) -> "ScriptedJudge":
+        self._rubric = rubric
+        return self
+
+    def set_aggregate(self, spec_id: str, task_id: str, aggregate: float) -> "ScriptedJudge":
+        self._fixed[(spec_id, task_id)] = aggregate
+        return self
+
+    def set_base(self, aggregate: float) -> "ScriptedJudge":
+        self._base = aggregate
+        return self
+
+    def with_noise_band(self, width: float) -> "ScriptedJudge":
+        self._noise_width = width
+        return self
+
+    def regress_after(self, trigger_spec_id: str, drop: float) -> "ScriptedJudge":
+        """Arm regression: after `trigger_spec_id` is scored once, drop subsequent by `drop`."""
+
+        self._regress = (trigger_spec_id, drop)
+        self._regress_armed = False
+        return self
+
+    def raise_on_next(self, exc: Exception | type[Exception] | None = None) -> "ScriptedJudge":
+        """Force the next score() call to raise (default LLMError). Consumed once."""
+
+        self._raise_next = (exc if isinstance(exc, Exception) else (exc or LLMError)("scripted outage"))
+        return self
+
+    @property
+    def rubric(self) -> Rubric:
+        return self._rubric
+
+    # --------------------------------------------------------------- contract
+    def score(self, trace: m.Trace, task: Task, rubric_id: str) -> m.RunScore:
+        if self._raise_next is not None:
+            err = self._raise_next
+            self._raise_next = None
+            raise err
+
+        aggregate = self._resolve_aggregate(trace, task)
+        confidence = 1.0
+
+        rs = m.RunScore(
+            run_id=trace.run_id,
+            spec_id=trace.spec_id,
+            task_id=trace.task_id,
+            rubric_scores={dim: aggregate for dim in self._rubric.sub_rubrics},
+            aggregate=aggregate,
+            confidence=confidence,
+            judge_meta=m.JudgeMeta(model="scripted-judge", rubric_id=rubric_id),
+            step_scores=[
+                m.StepScore(
+                    node_id=step.node_id,
+                    sub_rubrics={dim: aggregate for dim in self._rubric.sub_rubrics},
+                    note=f"scripted score for {step.node_id}",
+                )
+                for step in trace.steps
+            ],
+        )
+        self.scored.append(rs)
+        return rs
+
+    def score_suite(
+        self, scores: Sequence[m.RunScore], *, suite_id: str, rubric_id: str
+    ) -> SuiteAggregate:
+        return aggregate_scores(scores, suite_id=suite_id, rubric_id=rubric_id)
+
+    # --------------------------------------------------------------- internals
+    def _resolve_aggregate(self, trace: m.Trace, task: Task) -> float:
+        # regression hook takes priority over fixed values (it models a real
+        # post-promotion regression regardless of scripted per-task scores)
+        if self._regress is not None:
+            trigger, drop = self._regress
+            if trace.spec_id == trigger and not self._regress_armed:
+                self._regress_armed = True
+            elif self._regress_armed and trace.spec_id != trigger:
+                # subsequent non-trigger specs regress
+                base = self._fixed.get((trace.spec_id, trace.task_id), self._base)
+                return max(0.0, base - drop)
+
+        base = self._fixed.get((trace.spec_id, trace.task_id), self._base)
+
+        # deterministic noise: index of this run within the recorded history,
+        # so identical runs -> identical jitter (reproducible E1 boundary tests)
+        if self._noise_width > 0:
+            idx = len(self.scored)
+            # map idx onto a small repeating pattern within [-width, +width]
+            pattern = list(SCRIPTED_NOISE_PATTERN)
+            jitter = (pattern[idx % len(pattern)]) * self._noise_width
+            base = max(0.0, min(1.0, base + jitter))
+        return base
+
+
+__all__ = ["ScriptedJudge"]

archforge/lint.py

@@ -0,0 +1,180 @@
+"""Spec Linter — pure structural validation of a Pipeline Spec (spec E5, I2).
+
+`lint(spec) -> list[LintError]` is a pure function (no I/O). It is reused in two
+places: inside the Architect (reject a candidate before it reaches the
+SuiteRunner — E5) and as the standalone `archforge lint` subcommand.
+
+The linter checks *structure*, not host behaviour:
+
+  * DAG-ness (no cycles, no self-loops)
+  * every edge endpoint resolves to an existing node
+  * edge-type rules (e.g. CONDITIONAL requires a `gate`)
+  * no orphan nodes in a multi-node spec; no duplicate node_ids
+  * tool ids are non-empty and unique within a node
+
+Tool ids are checked for well-formedness only — resolution against the host's
+tool registry is the host's job, not the linter's.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from typing import Literal
+
+from pydantic import BaseModel
+
+from archforge.models import EdgeType, NodeKind, Spec
+
+# Lint error codes — stable identifiers so tests and the Architect can branch on
+# them. Keep the set small; add codes only when a new structural fault appears.
+LintCode = Literal[
+    "duplicate_node",
+    "orphan_node",
+    "edge_unknown_from",
+    "edge_unknown_to",
+    "self_loop",
+    "duplicate_edge",
+    "cycle",
+    "conditional_no_gate",
+    "tool_id_empty",
+    "tool_id_duplicate",
+    "node_unknown_kind",
+]
+
+
+class LintError(BaseModel):
+    """A single structural defect in a Spec. Structured → returnable to humans."""
+
+    code: LintCode
+    message: str
+    location: str | None = None  # node_id or "from->to" depending on the fault
+
+
+def lint(spec: Spec) -> list[LintError]:
+    """Return all structural defects found in `spec` (empty list == valid)."""
+
+    errors: list[LintError] = []
+    node_ids = {n.node_id for n in spec.nodes}
+
+    # --- duplicate node_ids --------------------------------------------------
+    seen: dict[str, int] = defaultdict(int)
+    for n in spec.nodes:
+        seen[n.node_id] += 1
+        if seen[n.node_id] == 2:
+            errors.append(
+                LintError(code="duplicate_node", message=f"node_id '{n.node_id}' appears more than once",
+                          location=n.node_id)
+            )
+        # tool ids per node
+        tool_seen: set[str] = set()
+        for tid in n.tools:
+            if not tid:
+                errors.append(LintError(code="tool_id_empty",
+                                        message=f"node '{n.node_id}' has an empty tool id",
+                                        location=n.node_id))
+            elif tid in tool_seen:
+                errors.append(LintError(code="tool_id_duplicate",
+                                        message=f"node '{n.node_id}' lists tool '{tid}' more than once",
+                                        location=n.node_id))
+            else:
+                tool_seen.add(tid)
+        # node-kind rule (non-LLM optimizer extension). A bad `kind` value is
+        # normally rejected by pydantic's enum coercion at construction — this is
+        # a defense-in-depth guard for a model_construct / hand-edit path that
+        # bypassed validation. We deliberately do NOT require non-empty
+        # system_prompt/model on LLM nodes: this project's config-decay contract
+        # (`host/adapters/helpers.cfg_decay` -> `KnobVote`) treats an empty
+        # system_prompt as "the agent owns its prompt" and a None model as "the
+        # agent's hardcoded default". Requiring them would regress the established
+        # "text-in/text-out node need not declare one" SpecBuilder contract and
+        # break every legacy text-in/text-out Spec on lint — violating the design's
+        # own "legacy Specs load unchanged" back-compat guarantee.
+        if not isinstance(n.kind, NodeKind):
+            errors.append(LintError(code="node_unknown_kind",
+                                    message=f"node '{n.node_id}' has an unknown kind {n.kind!r}",
+                                    location=n.node_id))
+
+    # --- edge endpoints + per-edge rules ------------------------------------
+    edge_keys: set[tuple[str, str, str]] = set()
+    adjacency: dict[str, list[str]] = defaultdict(list)
+    indegree: dict[str, int] = {nid: 0 for nid in node_ids}
+
+    for edge in spec.edges:
+        f, t = edge.from_, edge.to
+        key = (f, t, edge.type.value)
+        if key in edge_keys:
+            errors.append(LintError(code="duplicate_edge",
+                                    message=f"duplicate edge {f}->{t} ({edge.type.value})",
+                                    location=f"{f}->{t}"))
+        edge_keys.add(key)
+
+        if f not in node_ids:
+            errors.append(LintError(code="edge_unknown_from",
+                                    message=f"edge from unknown node '{f}' (to '{t}')",
+                                    location=f"{f}->{t}"))
+        if t not in node_ids:
+            errors.append(LintError(code="edge_unknown_to",
+                                    message=f"edge to unknown node '{t}' (from '{f}')",
+                                    location=f"{f}->{t}"))
+        if f == t:
+            errors.append(LintError(code="self_loop",
+                                    message=f"edge {f}->{t} is a self-loop",
+                                    location=f"{f}->{t}"))
+        if edge.type is EdgeType.CONDITIONAL and not edge.gate:
+            errors.append(LintError(code="conditional_no_gate",
+                                    message=f"conditional edge {f}->{t} has no `gate`",
+                                    location=f"{f}->{t}"))
+
+        # Only count well-formed edges toward the graph (avoids KeyError spam).
+        if f in node_ids and t in node_ids and f != t:
+            adjacency[f].append(t)
+            indegree[t] += 1
+
+    # --- cycle detection (Kahn's algorithm over the well-formed graph) -------
+    if _has_cycle(node_ids, adjacency, indegree):
+        # If a self-loop already explained one cycle, keep both; both are real.
+        errors.append(LintError(code="cycle",
+                                message="execution graph contains a cycle",
+                                location=None))
+
+    # --- orphan nodes (a multi-node spec must wire every node) ---------------
+    if len(spec.nodes) > 1:
+        touched: set[str] = set()
+        for edge in spec.edges:
+            if edge.from_ in node_ids:
+                touched.add(edge.from_)
+            if edge.to in node_ids:
+                touched.add(edge.to)
+        for n in spec.nodes:
+            if n.node_id not in touched:
+                errors.append(LintError(code="orphan_node",
+                                        message=f"node '{n.node_id}' is not connected to any edge",
+                                        location=n.node_id))
+
+    return errors
+
+
+def is_valid(spec: Spec) -> bool:
+    """True iff the Spec has no structural defects."""
+
+    return not lint(spec)
+
+
+def _has_cycle(
+    node_ids: set[str],
+    adjacency: dict[str, list[str]],
+    indegree: dict[str, int],
+) -> bool:
+    """Kahn's: if we can't emit every node, a cycle exists."""
+
+    indeg = dict(indegree)  # copy; Kahn mutates
+    q: deque[str] = deque(n for n, d in indeg.items() if d == 0)
+    emitted = 0
+    while q:
+        n = q.popleft()
+        emitted += 1
+        for nxt in adjacency.get(n, []):
+            indeg[nxt] -= 1
+            if indeg[nxt] == 0:
+                q.append(nxt)
+    return emitted != len(node_ids)

archforge/llm/__init__.py

@@ -0,0 +1,65 @@
+"""Provider-agnostic LLM abstraction (spec §3 Judge/Architect).
+
+Both thinking components — the Judge and the Architect — talk
+to a model only through the `LLMClient` protocol. This is the **single seam**
+where "real vs fake" lives: tests plug in `ScriptedLLM` (deterministic, free,
+crashable); a run plugs in a real provider via `make_client(provider)`.
+
+The real provider adapters (`anthropic`, `openai`, `groq`, `gemini`) import their
+SDKs lazily inside `__init__`, so importing THIS package never requires any
+provider SDK to be installed — only an actual run with that provider does, at
+which point a missing SDK surfaces as a clear `LLMError` rather than a bare
+`ImportError`. The adapter classes are NOT re-exported here to keep the import
+graph SDK-free; `make_client(provider)` resolves them by name on demand.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from archforge.config import ALL_PROVIDERS, REAL_PROVIDERS
+from archforge.llm.base import (
+    Completion,
+    LLMClient,
+    LLMError,
+    Message,
+    Role,
+    Usage,
+)
+from archforge.llm.scripted import ScriptedLLM
+
+# `REAL_PROVIDERS` / `ALL_PROVIDERS` are re-exported (below in __all__) straight
+# from archforge.config — the single source of truth the CLI also imports.
+
+
+def make_client(provider: str, **kwargs: Any) -> LLMClient:
+    """Build a real `LLMClient` for `provider`.
+
+    Resolves the adapter lazily (importing the SDK is deferred to here). Raises
+    `LLMError` for an unknown provider so the CLI surfaces one clear message
+    across the provider seam. `kwargs` (api_key/base_url) pass straight through.
+    """
+
+    if provider == "scripted":
+        raise LLMError("'scripted' has no real client; use ScriptedLLM directly")
+    table = {
+        "anthropic": "archforge.llm.anthropic:AnthropicClient",
+        "openai": "archforge.llm.openai:OpenAIClient",
+        "groq": "archforge.llm.groq:GroqClient",
+        "gemini": "archforge.llm.gemini:GeminiClient",
+    }
+    ref = table.get(provider)
+    if ref is None:
+        raise LLMError(f"unknown LLM provider {provider!r}; expected one of {REAL_PROVIDERS}")
+    mod_name, cls_name = ref.split(":")
+    import importlib
+
+    module = importlib.import_module(mod_name)
+    cls = getattr(module, cls_name)
+    return cls(**kwargs)  # type: ignore[no-any-return]
+
+
+__all__ = [
+    "Completion", "LLMClient", "LLMError", "Message", "Role", "Usage", "ScriptedLLM",
+    "make_client", "REAL_PROVIDERS", "ALL_PROVIDERS",
+]