whetkit 0.1.0__py3-none-any.whl

whetkit/scoring/aggregate.py

@@ -0,0 +1,117 @@
+"""Aggregation: per-task scores and eval-level hit-rate metrics."""
+
+from pydantic import BaseModel
+
+from whetkit.datasets import TaskSpec
+from whetkit.llm import LLMProvider
+from whetkit.scoring.deterministic import MatchMode, ToolMatchResult, score_tool_match
+from whetkit.scoring.judge import JudgeCache, JudgeConfig, JudgeVerdict, judge_run
+from whetkit.tracing import TaskRun
+from whetkit.tracing.records import RunStatus
+
+
+class TaskScore(BaseModel):
+    task_id: str
+    run_status: RunStatus
+    tool_match: ToolMatchResult
+    judge: JudgeVerdict | None = None
+
+    @property
+    def tool_hit(self) -> bool:
+        return self.tool_match.matched
+
+    @property
+    def hit(self) -> bool:
+        """The headline metric: right tools AND (when judged) task success."""
+        if self.judge is not None and not self.judge.passed:
+            return False
+        return self.tool_hit
+
+
+class EvalSummary(BaseModel):
+    scores: list[TaskScore]
+
+    @property
+    def task_count(self) -> int:
+        return len(self.scores)
+
+    @property
+    def hit_rate(self) -> float:
+        if not self.scores:
+            return 0.0
+        return sum(s.hit for s in self.scores) / len(self.scores)
+
+    @property
+    def tool_hit_rate(self) -> float:
+        if not self.scores:
+            return 0.0
+        return sum(s.tool_hit for s in self.scores) / len(self.scores)
+
+    @property
+    def judge_pass_rate(self) -> float | None:
+        judged = [s for s in self.scores if s.judge is not None]
+        if not judged:
+            return None
+        return sum(s.judge.passed for s in judged) / len(judged)
+
+    @property
+    def avg_precision(self) -> float:
+        if not self.scores:
+            return 0.0
+        return sum(s.tool_match.precision for s in self.scores) / len(self.scores)
+
+    @property
+    def avg_recall(self) -> float:
+        if not self.scores:
+            return 0.0
+        return sum(s.tool_match.recall for s in self.scores) / len(self.scores)
+
+    def summary_lines(self) -> list[str]:
+        lines = [
+            f"Tasks: {self.task_count}",
+            f"Hit-rate: {self.hit_rate:.0%}",
+            f"Tool-selection hit-rate: {self.tool_hit_rate:.0%}",
+            f"Tool precision (avg): {self.avg_precision:.0%}",
+            f"Tool recall (avg): {self.avg_recall:.0%}",
+        ]
+        if (rate := self.judge_pass_rate) is not None:
+            lines.insert(2, f"Judge pass-rate: {rate:.0%}")
+        return lines
+
+
+async def score_runs(
+    tasks: list[TaskSpec],
+    runs: list[TaskRun],
+    mode: MatchMode = MatchMode.ORDER_TOLERANT,
+    judge_config: JudgeConfig | None = None,
+    judge_provider: LLMProvider | None = None,
+    judge_cache: JudgeCache | None = None,
+    use_judge: bool = False,
+) -> EvalSummary:
+    """Score each run against its task. Runs and tasks are matched by id;
+    a task with no run is scored as a total miss."""
+    runs_by_task = {run.task_id: run for run in runs}
+    scores: list[TaskScore] = []
+    for task in tasks:
+        run = runs_by_task.get(task.id)
+        if run is None:
+            scores.append(
+                TaskScore(
+                    task_id=task.id,
+                    run_status=RunStatus.ERROR,
+                    tool_match=score_tool_match(task, [], mode),
+                )
+            )
+            continue
+        verdict = None
+        if use_judge:
+            verdict = await judge_run(task, run, judge_config, judge_provider, judge_cache)
+        scores.append(
+            TaskScore(
+                task_id=task.id,
+                run_status=run.status,
+                tool_match=score_tool_match(task, run.called_tool_names, mode),
+                judge=verdict,
+            )
+        )
+    return EvalSummary(scores=scores)

whetkit/scoring/deterministic.py

@@ -0,0 +1,99 @@
+"""Deterministic tool-selection scoring.
+
+Compares the tools an agent actually called against a task's
+``expected_tools`` slots (each slot lists acceptable alternatives).
+
+Two modes:
+
+- ``order_tolerant`` (default): every slot must be satisfied by a distinct
+  call; extra calls are allowed (they cost precision, not the hit). When the
+  task sets ``ordered: true``, the satisfying calls must appear as a
+  subsequence of the call sequence.
+- ``exact``: the call sequence must be exactly one call per slot, in order,
+  with no extra calls.
+"""
+
+from enum import StrEnum
+
+from pydantic import BaseModel
+
+from whetkit.datasets import TaskSpec
+
+
+class MatchMode(StrEnum):
+    EXACT = "exact"
+    ORDER_TOLERANT = "order_tolerant"
+
+
+class ToolMatchResult(BaseModel):
+    matched: bool
+    mode: MatchMode
+    expected_slots: list[list[str]]
+    called: list[str]
+    satisfied_slots: int
+    missing_slots: list[list[str]]
+    extra_calls: list[str]
+    precision: float
+    recall: float
+
+
+def _assign_unordered(slots: list[list[str]], called: list[str]) -> list[int | None]:
+    """Greedily assign calls to slots ignoring order. Returns, per call, the
+    slot index it satisfied (or None if it satisfied nothing)."""
+    remaining = set(range(len(slots)))
+    assignment: list[int | None] = []
+    for name in called:
+        hit = next((i for i in sorted(remaining) if name in slots[i]), None)
+        if hit is not None:
+            remaining.discard(hit)
+        assignment.append(hit)
+    return assignment
+
+
+def _assign_ordered(slots: list[list[str]], called: list[str]) -> list[int | None]:
+    """Greedily match slots as a subsequence of the call sequence."""
+    next_slot = 0
+    assignment: list[int | None] = []
+    for name in called:
+        if next_slot < len(slots) and name in slots[next_slot]:
+            assignment.append(next_slot)
+            next_slot += 1
+        else:
+            assignment.append(None)
+    return assignment
+
+
+def score_tool_match(
+    task: TaskSpec,
+    called: list[str],
+    mode: MatchMode = MatchMode.ORDER_TOLERANT,
+) -> ToolMatchResult:
+    slots = task.expected_tool_slots
+
+    exact_ok = len(called) == len(slots) and all(
+        name in slot for name, slot in zip(called, slots, strict=True)
+    )
+    if exact_ok:
+        assignment: list[int | None] = list(range(len(called)))
+    else:
+        assignment = (_assign_ordered if task.ordered else _assign_unordered)(slots, called)
+
+    satisfied = {i for i in assignment if i is not None}
+    missing = [slot for i, slot in enumerate(slots) if i not in satisfied]
+    extras = [name for name, hit in zip(called, assignment, strict=True) if hit is None]
+
+    precision = (len(called) - len(extras)) / len(called) if called else 0.0
+    recall = len(satisfied) / len(slots) if slots else 1.0
+    matched = exact_ok if mode == MatchMode.EXACT else not missing
+
+    return ToolMatchResult(
+        matched=matched,
+        mode=mode,
+        expected_slots=slots,
+        called=called,
+        satisfied_slots=len(satisfied),
+        missing_slots=missing,
+        extra_calls=extras,
+        precision=precision,
+        recall=recall,
+    )

whetkit/scoring/judge.py

@@ -0,0 +1,170 @@
+"""LLM-as-judge grading of task success against the task's rubric.
+
+The judge sees the task, the rubric, a compact transcript of tool calls and
+results, and the agent's final answer. It returns a structured verdict.
+Judgments are cached in SQLite keyed by a content hash, so re-scoring the
+same run with the same judge model costs nothing.
+"""
+
+import hashlib
+import json
+import re
+import sqlite3
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from whetkit.datasets import TaskSpec
+from whetkit.llm import ChatMessage, LLMProvider, get_provider, parse_model
+from whetkit.tracing import TaskRun
+
+DEFAULT_JUDGE_MODEL = "anthropic:claude-sonnet-5"
+
+JUDGE_SYSTEM_PROMPT = """\
+You are a strict, impartial grader of AI agent runs. You are given a task a
+user asked an agent to do, the success criteria a correct run must satisfy,
+the tool calls the agent made (with results), and the agent's final answer.
+
+Grade ONLY against the success criteria:
+- If the criteria name specific facts or values, the final answer must state
+  them (equivalent phrasing and formatting are fine; wrong or missing values
+  are a fail).
+- Actions count only if the transcript shows they actually happened; the
+  agent claiming success without a supporting tool result is a fail.
+- Do not reward effort, verbosity, or plausible-sounding but unverified
+  claims. Do not penalize style.
+
+Respond with ONLY a JSON object, no markdown fences:
+{"passed": true or false, "rationale": "one or two sentences citing the evidence"}
+"""
+
+
+class JudgeVerdict(BaseModel):
+    passed: bool
+    rationale: str
+    judge_model: str
+    valid: bool = True  # False when the judge's output could not be parsed
+
+
+class JudgeConfig(BaseModel):
+    model: str = DEFAULT_JUDGE_MODEL
+    max_tokens: int = 512
+
+
+class JudgeCache:
+    """Content-addressed cache of judge verdicts (SQLite key/value)."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(self.path)
+        with self._conn:
+            self._conn.execute(
+                "CREATE TABLE IF NOT EXISTS judgments (key TEXT PRIMARY KEY, verdict TEXT NOT NULL)"
+            )
+
+    def get(self, key: str) -> JudgeVerdict | None:
+        row = self._conn.execute("SELECT verdict FROM judgments WHERE key = ?", (key,)).fetchone()
+        return JudgeVerdict.model_validate_json(row[0]) if row else None
+
+    def put(self, key: str, verdict: JudgeVerdict) -> None:
+        with self._conn:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO judgments (key, verdict) VALUES (?, ?)",
+                (key, verdict.model_dump_json()),
+            )
+
+    def close(self) -> None:
+        self._conn.close()
+
+
+def _transcript(run: TaskRun, max_result_chars: int = 500) -> str:
+    lines: list[str] = []
+    for turn in run.turns:
+        for call in turn.tool_calls:
+            result = call.result_text
+            if len(result) > max_result_chars:
+                result = result[:max_result_chars] + "…(truncated)"
+            status = "ERROR" if call.is_error else "ok"
+            lines.append(f"- {call.name}({json.dumps(call.arguments)}) [{status}] -> {result}")
+    return "\n".join(lines) or "(the agent made no tool calls)"
+
+
+def build_judge_prompt(task: TaskSpec, run: TaskRun) -> str:
+    return (
+        f"<task>\n{task.prompt}\n</task>\n\n"
+        f"<success_criteria>\n{task.success_criteria}\n</success_criteria>\n\n"
+        f"<tool_calls>\n{_transcript(run)}\n</tool_calls>\n\n"
+        f"<final_answer>\n{run.final_text or '(the agent never gave a final answer)'}\n"
+        f"</final_answer>"
+    )
+
+
+def _cache_key(task: TaskSpec, run: TaskRun, judge_model: str) -> str:
+    payload = json.dumps(
+        {
+            "judge_model": judge_model,
+            "task_id": task.id,
+            "criteria": task.success_criteria,
+            "prompt": build_judge_prompt(task, run),
+        },
+        sort_keys=True,
+    )
+    return hashlib.sha256(payload.encode()).hexdigest()
+
+
+def _parse_verdict(raw: str, judge_model: str) -> JudgeVerdict | None:
+    match = re.search(r"\{.*\}", raw, re.DOTALL)
+    if not match:
+        return None
+    try:
+        data = json.loads(match.group(0))
+        return JudgeVerdict(
+            passed=bool(data["passed"]),
+            rationale=str(data.get("rationale", "")),
+            judge_model=judge_model,
+        )
+    except (json.JSONDecodeError, KeyError, TypeError):
+        return None
+
+
+async def judge_run(
+    task: TaskSpec,
+    run: TaskRun,
+    config: JudgeConfig | None = None,
+    provider: LLMProvider | None = None,
+    cache: JudgeCache | None = None,
+) -> JudgeVerdict:
+    """Grade one run. Never raises on judge misbehavior: an unparseable
+    verdict comes back as ``passed=False, valid=False``."""
+    config = config or JudgeConfig()
+    key = _cache_key(task, run, config.model)
+    if cache is not None and (cached := cache.get(key)) is not None:
+        return cached
+
+    provider_name, model_id = parse_model(config.model)
+    provider = provider or get_provider(provider_name)
+
+    verdict: JudgeVerdict | None = None
+    for _attempt in range(2):
+        turn = await provider.complete(
+            model=model_id,
+            system=JUDGE_SYSTEM_PROMPT,
+            messages=[ChatMessage(role="user", content=build_judge_prompt(task, run))],
+            tools=[],
+            max_tokens=config.max_tokens,
+        )
+        verdict = _parse_verdict(turn.text or "", config.model)
+        if verdict is not None:
+            break
+    if verdict is None:
+        verdict = JudgeVerdict(
+            passed=False,
+            rationale="Judge output was not valid JSON after 2 attempts.",
+            judge_model=config.model,
+            valid=False,
+        )
+
+    if cache is not None and verdict.valid:
+        cache.put(key, verdict)
+    return verdict

whetkit/tracing/__init__.py

@@ -0,0 +1,21 @@
+"""Reasoning-path traces: structured records of every eval run."""
+
+from whetkit.tracing.records import TaskRun, ToolCallRecord, TurnRecord
+from whetkit.tracing.store import (
+    SCHEMA_VERSION,
+    TraceStore,
+    default_store_path,
+    read_jsonl,
+    write_jsonl,
+)
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "TaskRun",
+    "ToolCallRecord",
+    "TraceStore",
+    "TurnRecord",
+    "default_store_path",
+    "read_jsonl",
+    "write_jsonl",
+]

whetkit/tracing/records.py

@@ -0,0 +1,76 @@
+"""Structured records of an agent run: every turn, tool call, and outcome.
+
+These are the atoms the scorer, curator, and report all consume.
+"""
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from whetkit.llm.base import Usage
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+class RunStatus(StrEnum):
+    COMPLETED = "completed"  # model produced a final answer
+    MAX_TURNS = "max_turns"  # loop hit the turn limit before finishing
+    ERROR = "error"  # provider/transport failure aborted the run
+
+
+class ToolCallRecord(BaseModel):
+    """One tool invocation and its outcome."""
+
+    call_id: str
+    name: str
+    arguments: dict[str, Any] = {}
+    result_text: str = ""
+    is_error: bool = False
+    latency_ms: float = 0.0
+
+
+class TurnRecord(BaseModel):
+    """One model completion plus the tool calls it triggered."""
+
+    index: int
+    assistant_text: str | None = None
+    tool_calls: list[ToolCallRecord] = []
+    usage: Usage = Usage()
+    latency_ms: float = 0.0
+    stop_reason: str | None = None
+
+
+class TaskRun(BaseModel):
+    """A full agent run of one task against one server."""
+
+    task_id: str
+    server: str
+    model: str
+    started_at: datetime = Field(default_factory=utc_now)
+    finished_at: datetime | None = None
+    turns: list[TurnRecord] = []
+    final_text: str | None = None
+    status: RunStatus = RunStatus.COMPLETED
+    error: str | None = None
+
+    @property
+    def called_tool_names(self) -> list[str]:
+        """Every tool called, in call order across turns."""
+        return [call.name for turn in self.turns for call in turn.tool_calls]
+
+    @property
+    def total_usage(self) -> Usage:
+        total = Usage()
+        for turn in self.turns:
+            total = total + turn.usage
+        return total
+
+    @property
+    def total_latency_ms(self) -> float:
+        return sum(t.latency_ms for t in self.turns) + sum(
+            c.latency_ms for t in self.turns for c in t.tool_calls
+        )

whetkit/tracing/store.py

@@ -0,0 +1,172 @@
+"""Trace persistence: local SQLite (queryable) + JSONL (portable).
+
+The full pydantic record is the source of truth and is stored verbatim in
+``record_json``; the indexed columns exist for querying and reporting.
+The schema is versioned via the ``meta`` table — opening a store written by
+a newer schema fails loudly instead of corrupting it.
+
+Stage 1 is local-first by design: SQLite ships with CPython, needs no
+daemon, and one file per project is easy to inspect and delete.
+"""
+
+import json
+import sqlite3
+from collections.abc import Iterable
+from pathlib import Path
+
+from whetkit.tracing.records import TaskRun
+
+SCHEMA_VERSION = 1
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_group TEXT NOT NULL,
+    task_id TEXT NOT NULL,
+    server TEXT NOT NULL,
+    model TEXT NOT NULL,
+    status TEXT NOT NULL,
+    error TEXT,
+    started_at TEXT NOT NULL,
+    finished_at TEXT,
+    input_tokens INTEGER NOT NULL,
+    output_tokens INTEGER NOT NULL,
+    latency_ms REAL NOT NULL,
+    record_json TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_runs_group ON runs (run_group);
+CREATE INDEX IF NOT EXISTS idx_runs_task ON runs (task_id);
+"""
+
+
+class TraceStore:
+    """A local store of TaskRun traces, grouped by run label.
+
+    ``run_group`` names one eval batch (e.g. ``baseline`` or ``curated``) so
+    before/after comparisons can pull exactly the runs they need.
+    """
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(self.path)
+        self._conn.row_factory = sqlite3.Row
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        with self._conn:
+            self._conn.executescript(_SCHEMA)
+            row = self._conn.execute(
+                "SELECT value FROM meta WHERE key = 'schema_version'"
+            ).fetchone()
+            if row is None:
+                self._conn.execute(
+                    "INSERT INTO meta (key, value) VALUES ('schema_version', ?)",
+                    (str(SCHEMA_VERSION),),
+                )
+            elif int(row["value"]) > SCHEMA_VERSION:
+                raise RuntimeError(
+                    f"{self.path} uses trace schema v{row['value']}, but this "
+                    f"whetkit only understands up to v{SCHEMA_VERSION} — upgrade whetkit"
+                )
+
+    def save_run(self, run: TaskRun, run_group: str) -> int:
+        usage = run.total_usage
+        with self._conn:
+            cursor = self._conn.execute(
+                """
+                INSERT INTO runs (
+                    run_group, task_id, server, model, status, error,
+                    started_at, finished_at, input_tokens, output_tokens,
+                    latency_ms, record_json
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    run_group,
+                    run.task_id,
+                    run.server,
+                    run.model,
+                    str(run.status),
+                    run.error,
+                    run.started_at.isoformat(),
+                    run.finished_at.isoformat() if run.finished_at else None,
+                    usage.input_tokens,
+                    usage.output_tokens,
+                    run.total_latency_ms,
+                    run.model_dump_json(),
+                ),
+            )
+        assert cursor.lastrowid is not None
+        return cursor.lastrowid
+
+    def save_runs(self, runs: Iterable[TaskRun], run_group: str) -> list[int]:
+        return [self.save_run(run, run_group) for run in runs]
+
+    def load_runs(self, run_group: str | None = None) -> list[TaskRun]:
+        query = "SELECT record_json FROM runs"
+        params: tuple = ()
+        if run_group is not None:
+            query += " WHERE run_group = ?"
+            params = (run_group,)
+        query += " ORDER BY id"
+        rows = self._conn.execute(query, params).fetchall()
+        return [TaskRun.model_validate_json(row["record_json"]) for row in rows]
+
+    def list_groups(self) -> list[dict]:
+        rows = self._conn.execute(
+            """
+            SELECT run_group, COUNT(*) AS runs,
+                   MIN(started_at) AS first_started, MAX(started_at) AS last_started
+            FROM runs GROUP BY run_group ORDER BY MIN(id)
+            """
+        ).fetchall()
+        return [dict(row) for row in rows]
+
+    def close(self) -> None:
+        self._conn.close()
+
+    def __enter__(self) -> "TraceStore":
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+
+def write_jsonl(runs: Iterable[TaskRun], path: str | Path) -> None:
+    """Write one JSON record per line — greppable and diffable."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as f:
+        for run in runs:
+            f.write(run.model_dump_json() + "\n")
+
+
+def read_jsonl(path: str | Path) -> list[TaskRun]:
+    with Path(path).open() as f:
+        return [TaskRun.model_validate_json(line) for line in f if line.strip()]
+
+
+def default_store_path(base_dir: str | Path = ".") -> Path:
+    """Project-local default: ./.whetkit/traces.sqlite3 (gitignored)."""
+    return Path(base_dir) / ".whetkit" / "traces.sqlite3"
+
+
+def _summary_row(run: TaskRun) -> dict:
+    usage = run.total_usage
+    return {
+        "task_id": run.task_id,
+        "status": str(run.status),
+        "tools": run.called_tool_names,
+        "turns": len(run.turns),
+        "input_tokens": usage.input_tokens,
+        "output_tokens": usage.output_tokens,
+        "latency_ms": round(run.total_latency_ms, 1),
+    }
+
+
+def summarize_runs(runs: list[TaskRun]) -> str:
+    return json.dumps([_summary_row(run) for run in runs], indent=2)