debatebench 0.1.0__py3-none-any.whl

debatebench/__init__.py

@@ -0,0 +1,5 @@
+"""debatebench: structured, multi-turn adversarial LLM debates.
+
+Submodules are imported explicitly, never from here, so importing the backend
+seam doesn't pull in httpx or PyYAML (ADR-008).
+"""

debatebench/api.py

@@ -0,0 +1,242 @@
+"""The public API (ADR-028). Everything here is supported; everything else isn't.
+
+Two functions, named for the two commands:
+
+    import asyncio
+    from debatebench.api import debate, judge, load_run, write_transcript
+
+    config = load_run("run.yaml")
+    transcript = asyncio.run(debate(config))
+    write_transcript(transcript, config.output)
+
+    sheet = asyncio.run(judge(transcript, model="qwen3:8b",
+                              base_url="http://localhost:11434/v1", budget=6000))
+    print(sheet.winner, [side.total for side in sheet.sides])
+
+Nothing here writes a file on its own (ADR-028 §5) and nothing is synchronous
+(§6). Pass ``backends=`` or ``backend=`` and no HTTP call is made at all —
+anything with one ``async generate`` satisfies ADR-009's ``Backend``.
+
+This is the one module allowed to import both httpx and PyYAML, because
+composing the tool's two jobs needs both. That is why it is absent from
+tests/test_layering.py, and why it is not re-exported from ``__init__.py``.
+"""
+
+from __future__ import annotations
+
+# Aliased out of the namespace: this module's name list is its contract, so a
+# name that is here only to implement something must not read as an export.
+from collections.abc import Sequence as _Sequence
+from dataclasses import replace as _replace
+
+from .backend import (
+    DEFAULT_READ_TIMEOUT,
+    Backend,
+    BackendError,
+    GenerationRequest,
+    GenerationResult,
+    Message,
+)
+from .config import ConfigError, JudgeConfig, RunConfig, Side, Team, load_run
+from .event_stream import SCHEMA_VERSION as EVENT_SCHEMA_VERSION
+from .event_stream import make_event_writer
+from .events import DebateEvent, EventBus, EventType
+from .judging import (
+    DIMENSIONS,
+    HIT_STATUSES,
+    SCORE_SCHEMA_VERSION,
+    VERDICTS,
+    Claim,
+    DimensionScore,
+    FactCheck,
+    Hit,
+    JudgeError,
+    ScoreSheet,
+    SideScore,
+    write_scores,
+)
+from .judging import fact_check_debate as _fact_check_debate
+from .judging import score_debate as _score_debate
+from .judging import as_json_dict as scores_json
+from .openai_compat import OpenAICompatibleBackend, open_client
+from .orchestrator import DebateError
+from .orchestrator import run_debate as _orchestrate
+from .retrieval import RetrievalError
+from .transcript import SCHEMA_VERSION as TRANSCRIPT_SCHEMA_VERSION
+from .transcript import (
+    Evidence,
+    RunSnapshot,
+    SideSnapshot,
+    TeamSnapshot,
+    Transcript,
+    TranscriptError,
+    Turn,
+    Usage,
+    load_transcript,
+    write_transcript,
+)
+from .transcript import as_json_dict as transcript_json
+
+__all__ = [
+    # The two jobs (ADR-028 §2)
+    "debate",
+    "judge",
+    # Reading configs and transcripts
+    "load_run",
+    "load_transcript",
+    # Writing, which the two above deliberately do not do (ADR-028 §5)
+    "write_transcript",
+    "write_scores",
+    "transcript_json",
+    "scores_json",
+    # The backend seam: implement this and the API makes no HTTP call (ADR-009)
+    "Backend",
+    "GenerationRequest",
+    "GenerationResult",
+    "Message",
+    "OpenAICompatibleBackend",
+    "open_client",
+    "DEFAULT_READ_TIMEOUT",
+    # Watching a run in-process; --events is the out-of-process equivalent (ADR-027)
+    "DebateEvent",
+    "EventBus",
+    "EventType",
+    "make_event_writer",
+    # Config types
+    "RunConfig",
+    "JudgeConfig",
+    "Side",
+    "Team",
+    # Transcript types (ADR-005)
+    "Transcript",
+    "Turn",
+    "Usage",
+    "Evidence",
+    "RunSnapshot",
+    "SideSnapshot",
+    "TeamSnapshot",
+    # Score types (ADR-013, ADR-015)
+    "ScoreSheet",
+    "SideScore",
+    "DimensionScore",
+    "Hit",
+    "FactCheck",
+    "Claim",
+    # Every way this fails (ADR-028 Consequences: no common base, yet)
+    "ConfigError",
+    "DebateError",
+    "JudgeError",
+    "BackendError",
+    "TranscriptError",
+    "RetrievalError",
+    # The vocabularies a score file uses
+    "DIMENSIONS",
+    "HIT_STATUSES",
+    "VERDICTS",
+    # The three durable contracts (ADR-028 §9)
+    "TRANSCRIPT_SCHEMA_VERSION",
+    "SCORE_SCHEMA_VERSION",
+    "EVENT_SCHEMA_VERSION",
+]
+
+
+async def debate(
+    config: RunConfig,
+    *,
+    backends: _Sequence[Backend] | None = None,
+    events: EventBus | None = None,
+) -> Transcript:
+    """Run every configured phase and return the transcript. Writes nothing.
+
+    ``backends`` is one per side, in side-index order; omit it and one
+    ``OpenAICompatibleBackend`` per side is built from the config and torn down
+    before this returns, honouring ``config.timeout`` (ADR-025). Pass your own
+    and no HTTP client is opened, so ``config.timeout`` is not consulted — the
+    waiting is then your backend's business.
+
+    ``config.output`` is *not* used — ADR-007 requires the key, this function
+    ignores it (ADR-028 §5). Write the result yourself:
+    ``write_transcript(transcript, config.output)``.
+
+    Raises ``DebateError`` if any phase fails (Hard Rule 1: nothing partial
+    comes back), or ``BackendError`` if a server does.
+    """
+    if backends is not None:
+        _check_backends(backends, config)
+        return await _orchestrate(config, backends, events)
+    async with open_client(config.timeout) as client:
+        built = [OpenAICompatibleBackend(client, side.base_url, side.model) for side in config.sides]
+        return await _orchestrate(config, built, events)
+
+
+async def judge(
+    transcript: Transcript,
+    *,
+    model: str,
+    budget: int,
+    base_url: str | None = None,
+    backend: Backend | None = None,
+    fact_check: bool = True,
+    timeout: int | None = None,
+) -> ScoreSheet:
+    """Score a transcript, and audit its claims unless ``fact_check=False``.
+
+    One call for the rubric and, when the fact-check is on, a second for the
+    audit (ADR-015 §3). ``model`` is recorded in the sheet whichever backend
+    runs, so an injected one should be passed the name it is really using.
+
+    Pass ``base_url=`` to use the built-in adapter, or ``backend=`` for your
+    own. ``timeout`` applies only to a client this function opens, so it may not
+    be combined with ``backend=``. Writes nothing: ``write_scores(sheet, path)``
+    does that.
+
+    Raises ``JudgeError`` if the reply can't be parsed or overran its budget.
+    """
+    # Hard Rule 5: the budget is enforced here, not left to the caller's care.
+    # `debate` gets this from load_run's validation; these four arrive as bare
+    # keyword arguments, so this is the only place that can check them.
+    if budget < 1:
+        raise ValueError(f"judge() needs a budget of at least 1 completion token, got {budget}")
+    if not model.strip():
+        raise ValueError("judge() needs a non-empty model name: it is recorded in the score file")
+
+    if backend is not None:
+        if timeout is not None:
+            raise ValueError(
+                f"judge() was given both backend= and timeout={timeout}, and the timeout "
+                "would do nothing: it configures the HTTP client this function opens, and "
+                "your backend brings its own. Set the timeout on that backend instead"
+            )
+        return await _score(transcript, backend, model=model, budget=budget, fact_check=fact_check)
+    if base_url is None:
+        raise ValueError(
+            "judge() has nowhere to send the call: pass base_url= to use the built-in "
+            "openai-compatible adapter (e.g. 'http://localhost:11434/v1'), or backend= "
+            "with your own implementation of the Backend protocol (ADR-009)"
+        )
+    async with open_client(DEFAULT_READ_TIMEOUT if timeout is None else timeout) as client:
+        adapter = OpenAICompatibleBackend(client, base_url, model)
+        return await _score(transcript, adapter, model=model, budget=budget, fact_check=fact_check)
+
+
+async def _score(
+    transcript: Transcript, backend: Backend, *, model: str, budget: int, fact_check: bool
+) -> ScoreSheet:
+    """The scoring call, then the fact-check call when it's on (ADR-015 §3)."""
+    sheet = await _score_debate(
+        transcript, backend, model=model, budget=budget, fact_check_enabled=fact_check
+    )
+    if not fact_check:
+        return sheet
+    checked = await _fact_check_debate(transcript, backend, budget=budget)
+    return _replace(sheet, fact_check=checked)
+
+
+def _check_backends(backends: _Sequence[Backend], config: RunConfig) -> None:
+    """Caught here, not as an IndexError three turns into a run (Hard Rule 1)."""
+    if len(backends) != len(config.sides):
+        raise ValueError(
+            f"debate() was given {len(backends)} backend(s) for {len(config.sides)} sides; "
+            "pass one per side, in side-index order — backends[side.index] is the one "
+            "each side speaks through"
+        )

debatebench/backend.py

@@ -0,0 +1,56 @@
+"""The backend seam: one async method between the orchestrator and a model server.
+
+Types per ADR-009. The single-method async ``Protocol`` shape is a design lift
+from arbgjr/multi-agent-debate's ``LLMProviderProtocol`` (MIT; see ADR-001).
+This module imports only the standard library (ADR-008).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal, Protocol
+
+Role = Literal["system", "user", "assistant"]
+
+# ADR-025 §1: seconds to wait for a reply. It lives here because it is the one
+# module both `config` (which may not import httpx) and `openai_compat` (which
+# may not import yaml) are allowed to import — ADR-008's layering, pinned by
+# tests/test_layering.py. 600 was the hardcoded value before ADR-025 made it a
+# setting, and stays the default so no existing config behaves differently.
+DEFAULT_READ_TIMEOUT = 600
+
+
+@dataclass(frozen=True)
+class Message:
+    role: Role
+    content: str
+
+
+@dataclass(frozen=True)
+class GenerationRequest:
+    messages: tuple[Message, ...]
+    max_completion_tokens: int  # the per-phase budget
+    seed: int | None = None  # the run's seed, sent with every request (ADR-009)
+
+
+@dataclass(frozen=True)
+class GenerationResult:
+    """A reply exactly as the server gave it.
+
+    Empty text and a completion_tokens count over budget are returned as they
+    are: judging a reply is the orchestrator's job (Hard Rules 1 and 5).
+    """
+
+    text: str
+    prompt_tokens: int
+    completion_tokens: int
+    finish_reason: str
+    latency_ms: int
+
+
+class BackendError(Exception):
+    """No usable reply: an HTTP error, a timeout, or a malformed or usage-less reply."""
+
+
+class Backend(Protocol):
+    async def generate(self, request: GenerationRequest) -> GenerationResult: ...

debatebench/cli.py

@@ -0,0 +1,198 @@
+"""The ``debate`` command: a run.yaml, plus ADR-021's override flags.
+
+It runs every configured phase, then writes the transcript as JSON to the
+`output:` path, rotating any file already there to `<output>.1` (ADR-005).
+Nothing but that JSON goes to the file; everything the command says goes to
+stderr (Hard Rule 7).
+
+`--model` and `--budget` override the file for one run, so an A/B needs no
+second config (ADR-021). They change the loaded config before anything runs,
+which is what keeps the transcript's snapshot a record of what actually spoke.
+
+The run itself is `api.debate` (ADR-028 §4): this module is that function plus
+argv, stderr and an exit code, so a library caller and this command cannot get
+different behaviour out of the same config.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from collections.abc import Sequence
+from dataclasses import replace
+from typing import Any
+
+from .api import debate
+from .backend import BackendError
+from .config import ConfigError, RunConfig, Side, load_run
+from .event_stream import make_event_writer
+from .events import DebateEvent, EventBus, EventType, Listener
+from .orchestrator import DebateError
+from .transcript import write_transcript
+
+
+def _parser() -> argparse.ArgumentParser:
+    """One definition of the flags, so tests pin the names the CLI really uses."""
+    parser = argparse.ArgumentParser(
+        prog="debate",
+        description="Run a structured debate. Every setting, including the output path, "
+        "comes from the run.yaml file; --model and --budget override it for one run.",
+    )
+    parser.add_argument("run_yaml", metavar="run.yaml", help="the run's config file")
+    # ADR-021: model and budget only. Nothing else is overridable, on purpose —
+    # see its §6 for why output, seed, base_url and topic are not.
+    parser.add_argument("--model", help="override both sides' model for this run")
+    parser.add_argument("--budget", type=int, help="override both sides' per-phase cap")
+    parser.add_argument("--pro-model", help="override only the pro side's model")
+    parser.add_argument("--con-model", help="override only the con side's model")
+    parser.add_argument("--pro-budget", type=int, help="override only the pro side's cap")
+    parser.add_argument("--con-budget", type=int, help="override only the con side's cap")
+    # ADR-025 §5: not an experimental variable — it cannot change a token of the
+    # output, only whether the output arrives, so it sits outside ADR-021's list.
+    parser.add_argument("--timeout", type=int, help="seconds to wait for a reply (default 600)")
+    # ADR-027: a machine-readable view on the one stream this command leaves free.
+    parser.add_argument(
+        "--events",
+        action="store_true",
+        help="stream one JSON object per event to stdout, for another program to read",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+
+    try:
+        config = load_run(args.run_yaml)
+    except ConfigError as e:
+        _log(f"config error: {e}")
+        return 1
+    try:
+        config, overrides = _overridden(config, args)
+    except _OverrideError as e:
+        _log(str(e))
+        return 1
+    for line in overrides:
+        _log(line)
+    if config.seed_generated:
+        _log(f"run.yaml sets no seed; generated seed {config.seed}")
+    # Checked before the debate, so a missing directory can't waste a whole run.
+    if not config.output.parent.is_dir():
+        _log(f"config error: the output directory does not exist: {config.output.parent}")
+        return 1
+
+    events = EventBus()
+    events.subscribe(make_log_event(tuple(side.side for side in config.sides)))
+    if args.events:
+        # stdout, which Hard Rule 7 left free by putting the transcript in a named
+        # file rather than a redirect. The stderr log is unaffected (ADR-027 §1).
+        events.subscribe(make_event_writer(config))
+    try:
+        transcript = asyncio.run(debate(config, events=events))
+    except (DebateError, BackendError) as e:
+        _log(f"debate failed: {e}")
+        return 1
+
+    try:
+        backup = write_transcript(transcript, config.output)
+    except OSError as e:
+        _log(f"could not write the transcript to {config.output}: {e}")
+        return 1
+    if backup is not None:
+        _log(f"moved the previous transcript to {backup.name}")
+    _log(f"wrote {len(transcript.turns)} turns over {len(config.phases)} phases to {config.output}")
+    return 0
+
+
+class _OverrideError(Exception):
+    """An override flag that can't be used as given (ADR-021)."""
+
+
+def _overridden(
+    config: RunConfig, args: argparse.Namespace
+) -> tuple[RunConfig, tuple[str, ...]]:
+    """Apply ADR-021's override flags, before anything runs.
+
+    Overriding the *config* rather than the backend call is what keeps ADR-005's
+    snapshot honest: the transcript records the model that actually spoke, never
+    the one the file happened to name.
+    """
+    one_side = {
+        "model": {"pro": args.pro_model, "con": args.con_model},
+        "budget": {"pro": args.pro_budget, "con": args.con_budget},
+    }
+    both = {"model": args.model, "budget": args.budget}
+
+    # ADR-021 §4: no precedence rule, because any would discard something asked for.
+    for setting, shared in both.items():
+        if shared is None:
+            continue
+        clashing = [f"--{s}-{setting}" for s, v in one_side[setting].items() if v is not None]
+        if clashing:
+            raise _OverrideError(
+                f"--{setting} sets both sides and {' and '.join(clashing)} sets one; "
+                f"they conflict (ADR-021 §4). Drop one: --{setting} alone for a symmetric "
+                f"change, or --pro-{setting}/--con-{setting} alone for an asymmetric one"
+            )
+
+    changed: list[str] = []
+    sides: list[Side] = []
+    for side in config.sides:
+        values: dict[str, Any] = {}
+        for setting in ("model", "budget"):
+            picked = one_side[setting][side.side]
+            flag = f"--{side.side}-{setting}" if picked is not None else f"--{setting}"
+            new = picked if picked is not None else both[setting]
+            if new is None:
+                continue
+            _validate_override(setting, new, flag)
+            was = getattr(side, setting)
+            if new == was:
+                continue
+            values[setting] = new
+            changed.append(
+                f"{flag}: side {side.index} ({side.side}) {setting} {was!r} -> {new!r}"
+            )
+        sides.append(replace(side, **values) if values else side)
+
+    return replace(config, sides=(sides[0], sides[1])), tuple(changed)
+
+
+def _validate_override(setting: str, value: Any, flag: str) -> None:
+    """ADR-021 §5: a flag faces the check its file key does, named for the flag."""
+    if setting == "budget" and value < 1:
+        raise _OverrideError(f"{flag} must be an integer of at least 1, got {value}")
+    if setting == "model" and not value.strip():
+        raise _OverrideError(f"{flag} must be a non-empty model name")
+
+
+def make_log_event(labels: tuple[str, ...]) -> Listener:
+    """The CLI's own view of a run, through the same seam a dashboard would use.
+
+    ``labels`` is each side's ``pro``/``con`` from run.yaml (ADR-007 §7). A bare
+    index makes the reader hold the mapping in their head for the whole run, so a
+    turn names its side the way ``judge`` already does: ``side 0 (pro)``.
+    """
+
+    def log_event(event: DebateEvent) -> None:
+        if event.type is EventType.PHASE_STARTED:
+            _log(f"phase {event.phase_index}: {event.phase}")
+        elif event.type is EventType.TURN_COMPLETED and event.turn is not None:
+            turn = event.turn
+            # B2 only reports a reply that reached its budget; ADR-010 keeps it a valid turn.
+            capped = " (hit budget)" if turn.hit_budget else ""
+            _log(
+                f"  side {turn.side_index} ({labels[turn.side_index]}), "
+                f"spoke {'first' if turn.order == 0 else 'second'}: "
+                f"{turn.usage.completion_tokens} of {turn.budget} completion tokens{capped}, "
+                f"{turn.usage.prompt_tokens} prompt tokens, {turn.latency_ms} ms"
+            )
+            for line in turn.text.strip().splitlines():
+                _log(f"    | {line}")
+
+    return log_event
+
+
+def _log(message: str) -> None:
+    print(f"debate: {message}", file=sys.stderr)