thinkless 0.2.0__py3-none-any.whl

thinkless/bench/intents_report.py

@@ -0,0 +1,87 @@
+"""Reports for the intents benchmark."""
+
+from __future__ import annotations
+
+from rich.table import Table
+
+from .intents import CascadePoint, IntentsBenchmark
+
+__all__ = ["cascade_highlights", "intents_markdown", "intents_table"]
+
+
+def _pct(value: float | None) -> str:
+    return "n/a" if value is None else f"{value * 100:.1f}%"
+
+
+def cascade_highlights(result: IntentsBenchmark) -> list[CascadePoint]:
+    """A few informative cascade operating points, lowest threshold first."""
+    wanted = (0.5, 0.7, 0.8, 0.9, 0.95)
+    return [p for p in result.cascade if p.threshold in wanted]
+
+
+def intents_table(result: IntentsBenchmark) -> Table:
+    table = Table(title=f"{result.dataset_title}: {result.examples} examples")
+    for column in (
+        "provider",
+        "accuracy",
+        "ECE",
+        "p50 latency",
+        "p95 latency",
+        f"threshold for {result.target_accuracy:.0%}",
+    ):
+        table.add_column(column, justify="right")
+    for name, report in result.providers.items():
+        rec = report.recommended
+        table.add_row(
+            name,
+            _pct(report.accuracy),
+            "n/a" if report.ece is None else f"{report.ece:.3f}",
+            f"{report.latency_p50_ms:.1f} ms",
+            f"{report.latency_p95_ms:.1f} ms",
+            f"{rec.threshold:.2f} (covers {_pct(rec.coverage)})" if rec else "not reached",
+        )
+    return table
+
+
+def intents_markdown(result: IntentsBenchmark) -> str:
+    lines = [
+        f"# Intent benchmark: {result.dataset}",
+        "",
+        f"- Dataset: {result.dataset_title} ([source]({result.source}))",
+        f"- Sample: {result.examples} test examples, seed {result.seed}, {result.labels} labels",
+        f"- LLM: `{result.llm}`",
+        f"- Environment: {result.environment.get('device', 'cpu')}, torch {result.environment.get('torch', 'n/a')}",
+        f"- ThinkLess {result.thinkless_version}, run {result.created_at}",
+        "",
+        "## Providers",
+        "",
+        f"| Provider | Accuracy | ECE | p50 latency | p95 latency | Cost per 1k | Threshold for {result.target_accuracy:.0%} accuracy |",
+        "|---|---:|---:|---:|---:|---:|---|",
+    ]
+    for name, report in result.providers.items():
+        rec = report.recommended
+        lines.append(
+            f"| `{name}` | {_pct(report.accuracy)} | {'n/a' if report.ece is None else f'{report.ece:.3f}'} | "
+            f"{report.latency_p50_ms:.1f} ms | {report.latency_p95_ms:.1f} ms | ${report.cost_per_1k:.4f} | "
+            + (f"{rec.threshold:.2f}, answers {_pct(rec.coverage)} alone" if rec else "not reached")
+            + " |"
+        )
+    if result.cascade:
+        lines += [
+            "",
+            "## Cascade",
+            "",
+            "Small models are tried in order; an answer below the threshold goes to the next one, and "
+            "finally to the LLM. Accuracy is for the whole cascade.",
+            "",
+            "| Threshold | Accuracy | Calls reaching the LLM | Mean latency | Cost per 1k | Answered by |",
+            "|---:|---:|---:|---:|---:|---|",
+        ]
+        for point in result.cascade:
+            answered = ", ".join(f"{k} {_pct(v)}" for k, v in point.answered_by.items() if v)
+            label = "never accept" if point.threshold > 1 else f"{point.threshold:.2f}"
+            lines.append(
+                f"| {label} | {_pct(point.accuracy)} | {_pct(point.llm_share)} | {point.mean_latency_ms:.0f} ms | "
+                f"${point.cost_per_1k:.4f} | {answered} |"
+            )
+    return "\n".join(lines) + "\n"

thinkless/bench/metrics.py

@@ -0,0 +1,115 @@
+"""Metrics shared by the benchmarks and the calibration tool."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+
+from pydantic import BaseModel
+
+__all__ = [
+    "ThresholdPoint",
+    "expected_calibration_error",
+    "percentile",
+    "recommend_threshold",
+    "threshold_sweep",
+]
+
+
+def percentile(values: Sequence[float], q: float) -> float:
+    """Linear-interpolated percentile, ``q`` in [0, 100]."""
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return float(ordered[0])
+    rank = (len(ordered) - 1) * q / 100.0
+    low, high = math.floor(rank), math.ceil(rank)
+    if low == high:
+        return float(ordered[low])
+    return float(ordered[low] + (ordered[high] - ordered[low]) * (rank - low))
+
+
+def expected_calibration_error(
+    confidences: Sequence[float], correct: Sequence[bool], bins: int = 10
+) -> float:
+    """ECE: the gap between confidence and accuracy, averaged over equal-width bins."""
+    if not confidences:
+        return 0.0
+    total = len(confidences)
+    error = 0.0
+    for index in range(bins):
+        low, high = index / bins, (index + 1) / bins
+        members = [
+            i
+            for i, c in enumerate(confidences)
+            if (low <= c < high) or (index == bins - 1 and c == 1.0)
+        ]
+        if not members:
+            continue
+        accuracy = sum(correct[i] for i in members) / len(members)
+        confidence = sum(confidences[i] for i in members) / len(members)
+        error += abs(accuracy - confidence) * len(members) / total
+    return error
+
+
+class ThresholdPoint(BaseModel):
+    """What happens at one threshold.
+
+    Attributes:
+        threshold: Minimum confidence to accept the model's answer.
+        coverage: Share of inputs the model answers on its own.
+        accuracy: Accuracy on the inputs it answers.
+        cascade_accuracy: Accuracy of the whole cascade when the rest go to
+            the fallback (only when fallback predictions are supplied).
+    """
+
+    threshold: float
+    coverage: float
+    accuracy: float | None
+    cascade_accuracy: float | None = None
+
+
+def threshold_sweep(
+    confidences: Sequence[float],
+    correct: Sequence[bool],
+    fallback_correct: Sequence[bool] | None = None,
+    thresholds: Sequence[float] | None = None,
+) -> list[ThresholdPoint]:
+    """Coverage and accuracy of a model across thresholds.
+
+    With ``fallback_correct``, also reports the accuracy of a cascade that
+    sends every input below threshold to the fallback.
+    """
+    grid = thresholds or [round(x * 0.05, 2) for x in range(0, 20)] + [0.97, 0.99]
+    points = []
+    n = len(confidences)
+    for threshold in grid:
+        accepted = [i for i in range(n) if confidences[i] >= threshold]
+        coverage = len(accepted) / n if n else 0.0
+        accuracy = sum(correct[i] for i in accepted) / len(accepted) if accepted else None
+        cascade = None
+        if fallback_correct is not None and n:
+            hits = sum(
+                correct[i] if confidences[i] >= threshold else fallback_correct[i] for i in range(n)
+            )
+            cascade = hits / n
+        points.append(
+            ThresholdPoint(
+                threshold=threshold, coverage=coverage, accuracy=accuracy, cascade_accuracy=cascade
+            )
+        )
+    return points
+
+
+def recommend_threshold(
+    points: Sequence[ThresholdPoint], target_accuracy: float
+) -> ThresholdPoint | None:
+    """The lowest threshold whose accepted answers reach ``target_accuracy``.
+
+    Lowest threshold means the highest coverage, so the fewest escalations.
+    """
+    for point in sorted(points, key=lambda p: p.threshold):
+        if point.accuracy is not None and point.accuracy >= target_accuracy and point.coverage > 0:
+            return point
+    return None

thinkless/bench/report.py

@@ -0,0 +1,123 @@
+"""Human-readable reports for benchmark results."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from rich.table import Table
+
+from .support import ModeReport, SupportBenchmark
+
+__all__ = ["support_markdown", "support_table"]
+
+
+def _pct(value: float | None) -> str:
+    return "n/a" if value is None else f"{value * 100:.1f}%"
+
+
+def _ms(value: float) -> str:
+    return f"{value / 1000:.2f} s" if value >= 1000 else f"{value:.0f} ms"
+
+
+ROWS: Sequence[tuple[str, Any]] = (
+    ("Task success (correct action)", lambda m: _pct(m.task_success)),
+    ("Intent accuracy", lambda m: _pct(m.intent_accuracy)),
+    ("Order id accuracy", lambda m: _pct(m.order_id_accuracy)),
+    ("LLM calls per ticket", lambda m: f"{m.llm_calls:.2f}"),
+    ("  for decisions", lambda m: f"{m.decision_llm_calls:.2f}"),
+    ("  for replies", lambda m: f"{m.generation_calls:.2f}"),
+    ("Decision time per ticket", lambda m: _ms(m.decision_ms)),
+    ("Reply generation time per ticket", lambda m: _ms(m.generation_ms)),
+    ("End-to-end latency p50", lambda m: _ms(m.latency_p50_ms)),
+    ("End-to-end latency p95", lambda m: _ms(m.latency_p95_ms)),
+    (
+        "LLM tokens per ticket (in / out)",
+        lambda m: f"{m.llm_input_tokens:.0f} / {m.llm_output_tokens:.0f}",
+    ),
+    ("LLM tokens spent on decisions", lambda m: f"{m.decision_llm_tokens:.0f}"),
+    (
+        "Billed cost per 1k tickets",
+        lambda m: f"${m.cost_usd * 1000:.3f}" if m.cost_usd else "n/a (local)",
+    ),
+    ("Reference cost per 1k tickets", lambda m: f"${m.reference_cost_per_1k_tickets:.2f}"),
+    ("Replies passing the grounding check", lambda m: _pct(m.grounded_rate)),
+)
+
+
+def _planes(mode: ModeReport) -> str:
+    total = sum(mode.decisions_by_plane.values()) or 1
+    order = ("rule", "model", "llm", "unresolved")
+    return ", ".join(
+        f"{plane} {mode.decisions_by_plane.get(plane, 0) / total * 100:.0f}%"
+        for plane in order
+        if mode.decisions_by_plane.get(plane)
+    )
+
+
+def support_table(bench: SupportBenchmark) -> Table:
+    """A Rich table comparing modes side by side."""
+    table = Table(
+        title=f"Support benchmark ({bench.modes[next(iter(bench.modes))].tickets} tickets)"
+    )
+    table.add_column("metric", style="dim")
+    for name in bench.modes:
+        table.add_column(name, justify="right")
+    for label, fn in ROWS:
+        table.add_row(label, *(fn(mode) for mode in bench.modes.values()))
+    table.add_row("Decisions by plane", *(_planes(m) for m in bench.modes.values()))
+    return table
+
+
+def support_markdown(bench: SupportBenchmark) -> str:
+    """The benchmark as a Markdown report."""
+    modes = list(bench.modes.values())
+    lines = [
+        "# Support benchmark",
+        "",
+        f"- ThinkLess {bench.thinkless_version}, run {bench.created_at}",
+        f"- Reasoning model: `{bench.reasoning_model}`",
+        f"- Engine threshold: {bench.threshold}",
+        f"- Reference price for cost estimates: `{bench.reference_price}`",
+        f"- Environment: {bench.environment.get('device', 'cpu')}, Python {bench.environment.get('python')}, "
+        f"torch {bench.environment.get('torch', 'n/a')}",
+        f"- Tickets: {modes[0].tickets}",
+        "- Billed cost is what the provider reported for each call (OpenRouter does). Reference cost "
+        "prices the same LLM tokens at the reference model's published rates, so runs on different "
+        "models and local runs can be compared; tokenizers differ, so compare it by ratio.",
+        "",
+        "| Metric | " + " | ".join(f"`{m.mode}`" for m in modes) + " |",
+        "|---|" + "---:|" * len(modes),
+    ]
+    for label, fn in ROWS:
+        lines.append(f"| {label.strip()} | " + " | ".join(fn(m) for m in modes) + " |")
+    lines.append("| Decisions by plane | " + " | ".join(_planes(m) for m in modes) + " |")
+    lines.append("")
+
+    for mode in modes:
+        lines += [
+            f"## Questions in `{mode.mode}` mode",
+            "",
+            "| Question | Answered by | Escalation rate | Accuracy (labeled) |",
+            "|---|---|---:|---:|",
+        ]
+        for name, stats in mode.questions.items():
+            answered = ", ".join(
+                f"{k} {v}" for k, v in sorted(stats["answered_by"].items(), key=lambda kv: -kv[1])
+            )
+            accuracy = _pct(stats["accuracy"]) + (
+                f" of {stats['labeled']}" if stats["labeled"] else ""
+            )
+            lines.append(
+                f"| `{name}` | {answered} | {_pct(stats['escalation_rate'])} | {accuracy} |"
+            )
+        lines.append("")
+        if mode.failures:
+            lines += [f"Failures in `{mode.mode}`:", ""]
+            for failure in mode.failures:
+                lines.append(
+                    f"- {failure['ticket_id']}: expected `{failure['expected']}`, got `{failure['got']}` "
+                    f"(intent `{failure['intent']}` from {failure['intent_provider']})"
+                )
+            lines.append("")
+    return "\n".join(lines)

thinkless/bench/support.py

@@ -0,0 +1,342 @@
+"""The support benchmark: the same agent and tickets, one run per decision-plane mode."""
+
+from __future__ import annotations
+
+import json
+import platform
+import time
+from collections import Counter, defaultdict
+from collections.abc import Callable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from .._version import __version__
+from ..decision import Usage
+from ..engine import Engine
+from ..pricing import PriceTable, default_prices
+from ..tracing import JSONLSink, MemorySink, Tracer
+from ..tracing.summary import TraceSummary
+from .metrics import percentile
+
+__all__ = ["ModeReport", "SupportBenchmark", "TicketResult", "run_support_benchmark"]
+
+# For these yes/no questions the expected action implies the true answer.
+LABELED_YES_NO: dict[str, Callable[[Mapping[str, Any]], bool]] = {
+    "wants_human": lambda expected: expected["action"] == "escalate_human",
+    "injection": lambda expected: expected["action"] == "escalate_security",
+}
+
+
+class TicketResult(BaseModel):
+    """One ticket processed in one mode."""
+
+    ticket_id: str
+    mode: str
+    expected_action: str
+    action: str
+    correct: bool
+    expected_intent: str | None = None
+    intent: Any = None
+    intent_provider: str | None = None
+    expected_order_id: str | None = None
+    order_id: str | None = None
+    reply_source: str
+    grounded: bool
+    decisions: dict[str, Any] = Field(default_factory=dict)
+    escalated: list[str] = Field(default_factory=list)
+    decision_llm_calls: int = 0
+    decision_llm_tokens: int = 0
+    decision_llm_ms: float = 0.0
+    summary: TraceSummary
+
+
+class ModeReport(BaseModel):
+    """Aggregates for one mode. Per-ticket values are means."""
+
+    mode: str
+    tickets: int
+    task_success: float
+    intent_accuracy: float | None
+    order_id_accuracy: float | None
+    llm_calls: float
+    decision_llm_calls: float
+    generation_calls: float
+    decisions_by_plane: dict[str, int]
+    escalations: float
+    latency_p50_ms: float
+    latency_p95_ms: float
+    decision_ms: float
+    generation_ms: float
+    llm_input_tokens: float
+    llm_output_tokens: float
+    decision_llm_tokens: float
+    cost_usd: float
+    reference_cost_per_1k_tickets: float
+    grounded_rate: float
+    template_fallbacks: int
+    questions: dict[str, dict[str, Any]]
+    failures: list[dict[str, Any]]
+
+
+class SupportBenchmark(BaseModel):
+    """A complete benchmark run, as written to ``results.json``."""
+
+    thinkless_version: str = __version__
+    created_at: str
+    reasoning_model: str
+    reference_price: str
+    threshold: float
+    environment: dict[str, Any]
+    modes: dict[str, ModeReport]
+    tickets: list[TicketResult]
+
+
+def environment_info(extra: Mapping[str, Any] | None = None) -> dict[str, Any]:
+    """Python, platform and accelerator, recorded with every result file."""
+    env: dict[str, Any] = {"python": platform.python_version(), "platform": platform.platform()}
+    try:
+        import torch
+
+        env["torch"] = torch.__version__
+        env["device"] = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu"
+    except ImportError:
+        env["device"] = "cpu"
+    env.update(extra or {})
+    return env
+
+
+def _truth(name: str, expected: Mapping[str, Any]) -> tuple[bool, Any]:
+    if name == "intent":
+        return expected.get("intent") is not None, expected.get("intent")
+    if name in LABELED_YES_NO:
+        return True, LABELED_YES_NO[name](expected)
+    if name == "order":
+        return "order_id" in expected, expected.get("order_id")
+    return False, None
+
+
+def _question_stats(
+    results: Sequence[TicketResult], scenarios: Mapping[str, Mapping[str, Any]]
+) -> dict[str, dict[str, Any]]:
+    providers: dict[str, Counter[str]] = defaultdict(Counter)
+    asked: Counter[str] = Counter()
+    escalated: Counter[str] = Counter()
+    hits: Counter[str] = Counter()
+    labeled: Counter[str] = Counter()
+    for result in results:
+        expected = scenarios[result.ticket_id]["expected"]
+        for name, view in result.decisions.items():
+            asked[name] += 1
+            providers[name][view.get("provider") or "none"] += 1
+            escalated[name] += int(name in result.escalated)
+            has_label, truth = _truth(name, expected)
+            if not has_label:
+                continue
+            value = view.get("value")
+            if name == "order":
+                value = value.get("order_id") if isinstance(value, Mapping) else None
+            labeled[name] += 1
+            hits[name] += int(value == truth)
+    return {
+        name: {
+            "asked": asked[name],
+            "answered_by": dict(providers[name]),
+            "escalation_rate": round(escalated[name] / asked[name], 4),
+            "accuracy": round(hits[name] / labeled[name], 4) if labeled[name] else None,
+            "labeled": labeled[name],
+        }
+        for name in asked
+    }
+
+
+def _mode_report(
+    mode: str,
+    results: Sequence[TicketResult],
+    scenarios: Mapping[str, Mapping[str, Any]],
+    reference: tuple[str, str],
+    prices: PriceTable,
+) -> ModeReport:
+    n = len(results)
+    summaries = [r.summary for r in results]
+    planes: Counter[str] = Counter()
+    for summary in summaries:
+        planes.update(summary.decisions_by_plane)
+    intent_rows = [r for r in results if r.expected_intent is not None and "intent" in r.decisions]
+    order_rows = [r for r in results if "order" in r.decisions]
+    llm_in = sum(s.llm_input_tokens for s in summaries)
+    llm_out = sum(s.llm_output_tokens for s in summaries)
+    reference_cost, _ = prices.cost(
+        reference[0], reference[1], Usage(input_tokens=llm_in, output_tokens=llm_out)
+    )
+    small_model_ms = sum(
+        s.time_by_plane_ms.get("model", 0.0) + s.time_by_plane_ms.get("rule", 0.0)
+        for s in summaries
+    )
+    decision_llm_ms = sum(r.decision_llm_ms for r in results)
+    all_llm_ms = sum(s.time_by_plane_ms.get("llm", 0.0) for s in summaries)
+
+    def mean(total: float, digits: int = 3) -> float:
+        return round(total / n, digits) if n else 0.0
+
+    return ModeReport(
+        mode=mode,
+        tickets=n,
+        task_success=mean(sum(r.correct for r in results), 4),
+        intent_accuracy=round(
+            sum(r.intent == r.expected_intent for r in intent_rows) / len(intent_rows), 4
+        )
+        if intent_rows
+        else None,
+        order_id_accuracy=round(
+            sum(r.order_id == r.expected_order_id for r in order_rows) / len(order_rows), 4
+        )
+        if order_rows
+        else None,
+        llm_calls=mean(sum(s.llm_calls for s in summaries)),
+        decision_llm_calls=mean(sum(r.decision_llm_calls for r in results)),
+        generation_calls=mean(sum(s.generation_calls for s in summaries)),
+        decisions_by_plane=dict(planes),
+        escalations=mean(sum(s.escalations for s in summaries)),
+        latency_p50_ms=round(percentile([s.duration_ms for s in summaries], 50), 1),
+        latency_p95_ms=round(percentile([s.duration_ms for s in summaries], 95), 1),
+        decision_ms=mean(small_model_ms + decision_llm_ms, 1),
+        generation_ms=mean(all_llm_ms - decision_llm_ms, 1),
+        llm_input_tokens=mean(llm_in, 1),
+        llm_output_tokens=mean(llm_out, 1),
+        decision_llm_tokens=mean(sum(r.decision_llm_tokens for r in results), 1),
+        cost_usd=mean(sum(s.cost_usd for s in summaries), 8),
+        reference_cost_per_1k_tickets=round(reference_cost / n * 1000, 4) if n else 0.0,
+        grounded_rate=mean(sum(r.grounded for r in results), 4),
+        template_fallbacks=sum(r.reply_source == "template_fallback" for r in results),
+        questions=_question_stats(results, scenarios),
+        failures=[
+            {
+                "ticket_id": r.ticket_id,
+                "expected": r.expected_action,
+                "got": r.action,
+                "intent": r.decisions.get("intent", {}).get("value"),
+                "intent_provider": r.intent_provider,
+            }
+            for r in results
+            if not r.correct
+        ],
+    )
+
+
+def _trace_facts(spans: Sequence[Mapping[str, Any]]) -> tuple[list[str], int, int, float]:
+    """Escalated question names and LLM decision usage, read from a trace."""
+    escalated: set[str] = set()
+    calls = tokens = 0
+    ms = 0.0
+    for span in spans:
+        attrs = span.get("attributes") or {}
+        if span.get("kind") == "decide":
+            escalated.update(d["name"] for d in attrs.get("decisions") or [] if d.get("escalated"))
+        if span.get("kind") != "attempt":
+            continue
+        if span.get("plane") == "llm":
+            usage = attrs.get("usage") or {}
+            calls += 1
+            tokens += int(usage.get("input_tokens") or 0) + int(usage.get("output_tokens") or 0)
+            ms += float(span.get("duration_ms") or 0.0)
+    return sorted(escalated), calls, tokens, ms
+
+
+def run_support_benchmark(
+    engines: Mapping[str, Callable[[Tracer], Engine]],
+    *,
+    reasoning_model: str,
+    scenarios: Sequence[Mapping[str, Any]] | None = None,
+    output_dir: str | Path | None = None,
+    reference_price: str = "anthropic:claude-sonnet-5",
+    threshold: float = 0.8,
+    prices: PriceTable | None = None,
+    on_ticket: Callable[[TicketResult], None] | None = None,
+    environment: Mapping[str, Any] | None = None,
+) -> SupportBenchmark:
+    """Run every ticket through every mode.
+
+    Args:
+        engines: Mode name to a factory that builds that mode's engine around
+            the tracer it is given.
+        reasoning_model: Label of the LLM, recorded in the results.
+        scenarios: Tickets with expected outcomes. Defaults to the bundled set.
+        output_dir: Where to write ``results.json`` and one trace file per
+            ticket and mode. Nothing is written when omitted.
+        reference_price: ``provider:model`` whose price list turns LLM tokens
+            into an estimated dollar cost, so local runs can be compared with
+            a hosted deployment.
+        threshold: The engine threshold, recorded in the results.
+        on_ticket: Progress callback.
+    """
+    from ..demo.support import SupportAgent, World, load_scenarios
+
+    tickets = list(scenarios or load_scenarios())
+    by_id = {str(t["id"]): t for t in tickets}
+    prices = prices or default_prices()
+    provider, _, model = reference_price.partition(":")
+    out = Path(output_dir) if output_dir else None
+
+    results: list[TicketResult] = []
+    for mode, factory in engines.items():
+        memory = MemorySink()
+        sinks: list[Any] = [memory]
+        if out is not None:
+            sinks.append(JSONLSink(out / "traces" / mode))
+        agent = SupportAgent(factory(Tracer(sinks)))
+        for ticket in tickets:
+            outcome, summary = agent.handle(
+                ticket, World(), mode=mode, expected_action=ticket["expected"]["action"]
+            )
+            escalated, calls, tokens, ms = _trace_facts(memory.trace(outcome.trace_id or ""))
+            memory.clear()
+            expected = ticket["expected"]
+            intent = outcome.decisions.get("intent") or {}
+            result = TicketResult(
+                ticket_id=str(ticket["id"]),
+                mode=mode,
+                expected_action=expected["action"],
+                action=outcome.action,
+                correct=outcome.action == expected["action"],
+                expected_intent=expected.get("intent"),
+                intent=intent.get("value") if intent.get("status") == "accepted" else None,
+                intent_provider=intent.get("provider"),
+                expected_order_id=expected.get("order_id"),
+                order_id=outcome.order_id,
+                reply_source=outcome.reply_source,
+                grounded=outcome.grounded,
+                decisions=outcome.decisions,
+                escalated=escalated,
+                decision_llm_calls=calls,
+                decision_llm_tokens=tokens,
+                decision_llm_ms=ms,
+                summary=summary,
+            )
+            results.append(result)
+            if on_ticket is not None:
+                on_ticket(result)
+
+    modes = {
+        mode: _mode_report(
+            mode, [r for r in results if r.mode == mode], by_id, (provider, model), prices
+        )
+        for mode in engines
+    }
+    benchmark = SupportBenchmark(
+        created_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        reasoning_model=reasoning_model,
+        reference_price=reference_price,
+        threshold=threshold,
+        environment=environment_info(environment),
+        modes=modes,
+        tickets=results,
+    )
+    if out is not None:
+        out.mkdir(parents=True, exist_ok=True)
+        (out / "results.json").write_text(
+            json.dumps(benchmark.model_dump(mode="json"), indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+    return benchmark

thinkless/cli/__init__.py

@@ -0,0 +1 @@
+"""Command line interface."""