eval-builder 0.1.0__py3-none-any.whl

eval_builder/judge/run.py

@@ -0,0 +1,149 @@
+"""Optional judge runner. OFF by default.
+
+eval-builder never calls a model provider and ships no API keys. This module only
+starts a command YOU name (for example a script that calls your local model or your
+provider), sends it one JSON request per line on stdin, and reads one JSON response
+per line from stdout:
+
+    request  (stdin):  the row from judge_requests.jsonl (includes "prompt" when the
+                       rubric judge has a prompt template)
+    response (stdout): {"verdict": "<label>", "raw": "<optional raw text>"}
+
+It refuses to run unless enabled with --enable-judge-plugin or the environment
+variable EVAL_BUILDER_ENABLE_JUDGE_PLUGIN=1.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import queue
+import shlex
+import subprocess
+import threading
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from ..io import read_json, read_jsonl, write_json
+from ..workspace import Workspace
+
+ENV_FLAG = "EVAL_BUILDER_ENABLE_JUDGE_PLUGIN"
+
+
+class PluginDisabled(PermissionError):
+    pass
+
+
+def plugin_enabled(flag: bool) -> bool:
+    return flag or os.environ.get(ENV_FLAG) == "1"
+
+
+class _LineProcess:
+    def __init__(self, argv: list[str]) -> None:
+        self.proc = subprocess.Popen(
+            argv,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+        self.lines: queue.Queue[str | None] = queue.Queue()
+        threading.Thread(target=self._pump, daemon=True).start()
+
+    def _pump(self) -> None:
+        assert self.proc.stdout is not None
+        for line in self.proc.stdout:
+            self.lines.put(line)
+        self.lines.put(None)
+
+    def ask(self, payload: dict[str, Any], timeout: float) -> dict[str, Any]:
+        assert self.proc.stdin is not None
+        self.proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n")
+        self.proc.stdin.flush()
+        try:
+            line = self.lines.get(timeout=timeout)
+        except queue.Empty as e:
+            raise TimeoutError(f"no response within {timeout}s") from e
+        if line is None:
+            err = self.proc.stderr.read() if self.proc.stderr else ""
+            raise RuntimeError(f"judge command exited: {err.strip()[-500:]}")
+        return json.loads(line)
+
+    def close(self) -> int:
+        if self.proc.stdin:
+            self.proc.stdin.close()
+        try:
+            return self.proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            self.proc.kill()
+            return -9
+
+
+def judge_run(
+    workspace: str | Path,
+    commands: dict[str, str],
+    enable: bool = False,
+    timeout: float = 120.0,
+    resume: bool = True,
+) -> dict[str, Any]:
+    if not plugin_enabled(enable):
+        raise PluginDisabled(
+            "the judge plugin is off by default; pass --enable-judge-plugin or set "
+            f"{ENV_FLAG}=1 to let eval-builder run the judge command you provide"
+        )
+    ws = Workspace.at(workspace)
+    if not ws.judge_requests.exists():
+        raise FileNotFoundError(f"{ws.judge_requests} not found; run `eval-builder judge-plan`")
+    requests = list(read_jsonl(ws.judge_requests))
+    done: set[str] = set()
+    if resume and ws.judgments.exists():
+        done = {r["request_id"] for r in read_jsonl(ws.judgments) if "request_id" in r}
+    elif ws.judgments.exists():
+        ws.judgments.unlink()
+    log: dict[str, Any] = {"started": datetime.now(UTC).isoformat(), "judges": {}}
+    with open(ws.judgments, "a", encoding="utf-8") as out:
+        for jid, cmd in commands.items():
+            todo = [r for r in requests if r["judge"] == jid and r["request_id"] not in done]
+            entry: dict[str, Any] = {
+                "command": cmd,
+                "requests": len(todo),
+                "ok": 0,
+                "errors": 0,
+                "seconds": 0.0,
+            }
+            log["judges"][jid] = entry
+            if not todo:
+                continue
+            t0 = time.monotonic()
+            proc = _LineProcess(shlex.split(cmd))
+            try:
+                for r in todo:
+                    row = {k: v for k, v in r.items() if k not in ("presented", "prompt")}
+                    try:
+                        resp = proc.ask(r, timeout)
+                        row["verdict"] = resp.get("verdict")
+                        # keep extra plugin fields (raw text, token counts) as evidence
+                        row.update({k: v for k, v in resp.items() if k not in row})
+                        entry["ok"] += 1
+                    except (TimeoutError, RuntimeError, json.JSONDecodeError) as e:
+                        row["verdict"] = None
+                        row["error"] = str(e)
+                        entry["errors"] += 1
+                        if isinstance(e, RuntimeError):
+                            out.write(json.dumps(row, ensure_ascii=False) + "\n")
+                            break
+                    out.write(json.dumps(row, ensure_ascii=False) + "\n")
+                    out.flush()
+            finally:
+                entry["exit_code"] = proc.close()
+                entry["seconds"] = round(time.monotonic() - t0, 2)
+    missing = sorted({r["judge"] for r in requests} - set(commands))
+    log["finished"] = datetime.now(UTC).isoformat()
+    log["judges_without_command"] = missing
+    log_path = ws.root / "judge_run_log.json"
+    runs = read_json(log_path).get("runs", []) if log_path.exists() else []
+    write_json(log_path, {"runs": [*runs, log]})
+    return log

eval_builder/judge/stats.py

@@ -0,0 +1,76 @@
+"""Plain statistics used by judge-check. Kept small so tests can verify them by hand."""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from collections.abc import Mapping, Sequence
+
+Z95 = 1.959963984540054
+
+
+def wilson(k: int, n: int, z: float = Z95) -> tuple[float, float] | None:
+    """Wilson score interval for a binomial proportion k/n."""
+    if n <= 0:
+        return None
+    p = k / n
+    denom = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / denom
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
+    return (max(0.0, centre - half), min(1.0, centre + half))
+
+
+def cohen_kappa(a: Sequence[str], b: Sequence[str]) -> tuple[float | None, float, float]:
+    """Return (kappa, observed agreement po, chance agreement pe). kappa is None if pe == 1."""
+    if len(a) != len(b):
+        raise ValueError("rating lists must have the same length")
+    n = len(a)
+    if n == 0:
+        return None, 0.0, 0.0
+    po = sum(x == y for x, y in zip(a, b, strict=True)) / n
+    ca, cb = Counter(a), Counter(b)
+    pe = sum(ca[label] * cb[label] for label in set(ca) | set(cb)) / (n * n)
+    if pe >= 1.0:
+        return None, po, pe
+    return (po - pe) / (1 - pe), po, pe
+
+
+def kappa_interval(po: float, pe: float, n: int, z: float = Z95) -> tuple[float, float] | None:
+    """Approximate 95% interval using Cohen's (1960) large-sample standard error."""
+    if n <= 0 or pe >= 1.0:
+        return None
+    kappa = (po - pe) / (1 - pe)
+    se = math.sqrt(po * (1 - po) / (n * (1 - pe) ** 2))
+    return (max(-1.0, kappa - z * se), min(1.0, kappa + z * se))
+
+
+def majority(labels: Sequence[str]) -> tuple[str | None, float]:
+    """Most common label and its share. Label is None when the top count is tied."""
+    if not labels:
+        return None, 0.0
+    counts = Counter(labels).most_common()
+    top, cnt = counts[0]
+    if len(counts) > 1 and counts[1][1] == cnt:
+        return None, cnt / len(labels)
+    return top, cnt / len(labels)
+
+
+def majority_vote_stability(counts: Mapping[str, int], k: int = 3) -> float | None:
+    """Chance that a majority vote over k calls (drawn without replacement from the
+    observed trials) returns the same label as the majority over all trials.
+
+    Exact hypergeometric probability that the full-majority label gets more than half
+    of the k draws. None when there are fewer than k + 2 trials (too few to say
+    anything beyond the full vote) or no strict majority.
+    """
+    n = sum(counts.values())
+    if n < k + 2 or not counts:
+        return None
+    ordered = sorted(counts.values(), reverse=True)
+    if len(ordered) > 1 and ordered[0] == ordered[1]:
+        return None
+    big = ordered[0]
+    need = k // 2 + 1
+    total = math.comb(n, k)
+    hits = sum(math.comb(big, x) * math.comb(n - big, k - x) for x in range(need, k + 1))
+    return hits / total

eval_builder/mcp_server.py

@@ -0,0 +1,175 @@
+"""MCP server (stdio). Every tool returns the same JSON the CLI prints with --json.
+
+The opt-in judge runner is deliberately not exposed here: it executes a command,
+which should stay an explicit CLI action by the user.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from . import draft as draft_mod
+from .export import export as export_fn
+from .ingest import ingest as ingest_fn
+from .judge.check import Thresholds
+from .judge.check import judge_check as judge_check_fn
+from .judge.plan import judge_plan as judge_plan_fn
+from .report import build_report
+from .select import select as select_fn
+from .status import status as status_fn
+
+INSTRUCTIONS = (
+    "eval-builder turns an app's logs into an eval suite and checks LLM judges. Workflow: "
+    "ingest -> select -> draft (you fill expected_behavior and criteria with the user, via "
+    "update_case and set_rubric) -> validate -> judge_plan (run each request several times "
+    "with the user's judge, save judgments.jsonl) -> judge_check -> export -> report. "
+    "Never invent human labels; ask the user for them."
+)
+
+
+def _server() -> Any:
+    try:
+        from mcp.server.mcpserver import MCPServer
+
+        return MCPServer("eval-builder", instructions=INSTRUCTIONS)
+    except ImportError:
+        from mcp.server.fastmcp import FastMCP  # type: ignore[attr-defined]  # mcp 1.x
+
+        return FastMCP("eval-builder", instructions=INSTRUCTIONS)
+
+
+def build_server() -> Any:
+    mcp = _server()
+
+    @mcp.tool()
+    def ingest(
+        paths: list[str], workspace: str = "evalset", format: str | None = None, redact: bool = True
+    ) -> dict[str, Any]:
+        """Read traces (OpenAI chat JSONL, Anthropic messages, Langfuse export, OpenTelemetry
+        GenAI spans, or generic input/output JSONL), normalize and redact them."""
+        return ingest_fn(paths, workspace, format, redact)
+
+    @mcp.tool()
+    def select(
+        workspace: str = "evalset",
+        n: int = 50,
+        seed: int = 0,
+        clusters: int | None = None,
+        failure_share: float = 0.3,
+        near_dup_threshold: float = 0.9,
+        dedupe_on: str = "input",
+        stratify: list[str] | None = None,
+    ) -> dict[str, Any]:
+        """Pick a diverse, representative set of traces and record why each was picked."""
+        r = select_fn(
+            workspace, n, seed, clusters, failure_share, near_dup_threshold, dedupe_on, stratify
+        )
+        return {k: v for k, v in r.items() if k != "selected"} | {
+            "selected": [
+                {"trace_id": s["trace_id"], "reasons": s["reasons"]} for s in r["selected"]
+            ]
+        }
+
+    @mcp.tool()
+    def draft(workspace: str = "evalset", suite: str = "eval-builder suite") -> dict[str, Any]:
+        """Write cases.yaml and rubric.yaml skeletons from the selection (keeps filled cases)."""
+        return draft_mod.draft(workspace, suite)
+
+    @mcp.tool()
+    def list_cases(workspace: str = "evalset", status: str | None = None) -> list[dict[str, Any]]:
+        """Return the cases in cases.yaml, optionally filtered by status."""
+        cases = draft_mod.load_cases(workspace).get("cases") or []
+        return [c for c in cases if status is None or c.get("status") == status]
+
+    @mcp.tool()
+    def update_case(
+        case_id: str,
+        workspace: str = "evalset",
+        expected_behavior: str | None = None,
+        criteria: list[str] | None = None,
+        reference_output: str | None = None,
+        compare_output: str | None = None,
+        status: str | None = None,
+        notes: str | None = None,
+    ) -> dict[str, Any]:
+        """Fill in or change one case. status is draft, ready or dropped."""
+        fields = {
+            "expected_behavior": expected_behavior,
+            "criteria": criteria,
+            "reference_output": reference_output,
+            "compare_output": compare_output,
+            "status": status,
+            "notes": notes,
+        }
+        return draft_mod.update_case(
+            workspace, case_id, **{k: v for k, v in fields.items() if v is not None}
+        )
+
+    @mcp.tool()
+    def set_rubric(
+        criteria: list[dict[str, Any]],
+        judges: list[dict[str, Any]] | None = None,
+        workspace: str = "evalset",
+    ) -> dict[str, Any]:
+        """Replace rubric.yaml. criteria: [{id, description, scale}]. judges: [{id, mode
+        (pointwise|pairwise), criteria, labels, prompt}]."""
+        return draft_mod.write_rubric(workspace, {"criteria": criteria, "judges": judges or []})
+
+    @mcp.tool()
+    def validate(workspace: str = "evalset") -> dict[str, Any]:
+        """Check cases.yaml and rubric.yaml; ready cases must have no TODO left."""
+        return draft_mod.validate(workspace)
+
+    @mcp.tool()
+    def judge_plan(
+        workspace: str = "evalset",
+        judges: list[str] | None = None,
+        trials: int = 5,
+        probes: list[str] | None = None,
+        probe_trials: int = 3,
+    ) -> dict[str, Any]:
+        """Write judge_requests.jsonl: every judge call to make, repeated `trials` times, plus
+        optional probes (swap: answer order, pairwise only; pad: irrelevant length)."""
+        return judge_plan_fn(workspace, judges, trials, probes, probe_trials)
+
+    @mcp.tool()
+    def judge_check(
+        workspace: str = "evalset",
+        judgments: str | None = None,
+        labels: str | None = None,
+        max_flip_rate: float = 0.2,
+        min_kappa: float = 0.4,
+        min_labeled: int = 20,
+        min_cases: int = 10,
+    ) -> dict[str, Any]:
+        """Flip rate, human agreement (accuracy, kappa), position and verbosity probes, and a
+        verdict per judge: trustworthy, unstable, biased, misaligned or not_enough_data."""
+        th = Thresholds(
+            max_flip_rate=max_flip_rate,
+            min_kappa=min_kappa,
+            min_labeled=min_labeled,
+            min_cases=min_cases,
+        )
+        r = judge_check_fn(workspace, judgments, labels, th)
+        return {k: v for k, v in r.items() if k != "judges"}
+
+    @mcp.tool()
+    def export(workspace: str = "evalset", formats: list[str] | None = None) -> dict[str, Any]:
+        """Export ready cases to promptfoo, deepeval, inspect and/or jsonl."""
+        return export_fn(workspace, formats)
+
+    @mcp.tool()
+    def report(workspace: str = "evalset", title: str | None = None) -> dict[str, Any]:
+        """Write report.md and report.json covering every step that has run."""
+        return build_report(workspace, title)
+
+    @mcp.tool()
+    def status(workspace: str = "evalset") -> dict[str, Any]:
+        """Which steps have run in this workspace and what to do next."""
+        return status_fn(workspace)
+
+    return mcp
+
+
+def main() -> None:
+    build_server().run("stdio")

eval_builder/redact.py

@@ -0,0 +1,101 @@
+"""Default-on redaction of obvious secrets and PII.
+
+Patterns are deliberately conservative: they catch common key formats, emails,
+card numbers that pass a Luhn check, US-style phone numbers with separators, and
+SSN-shaped numbers. They are not a DLP system; the report lists exactly what was
+replaced so you can check.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Any
+
+_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    (
+        "private_key",
+        re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY-----"),
+    ),
+    ("anthropic_key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
+    ("openai_key", re.compile(r"\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{20,}")),
+    ("aws_access_key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("github_token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})")),
+    ("slack_token", re.compile(r"\bxox[abposr]-[A-Za-z0-9-]{10,}")),
+    ("google_api_key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
+    ("stripe_key", re.compile(r"\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}")),
+    ("jwt", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")),
+    ("bearer_token", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/-]{20,}=*")),
+    ("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")),
+    ("ssn", re.compile(r"\b\d{3}-\d{2}-\d{4}\b")),
+    ("phone", re.compile(r"(?<![\d-])(?:\+1[\s.-]?)?\(?\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}(?![\d-])")),
+    ("credit_card", re.compile(r"\b\d(?:[ -]?\d){12,18}\b")),
+]
+
+
+def _luhn_ok(digits: str) -> bool:
+    total = 0
+    for i, ch in enumerate(reversed(digits)):
+        d = int(ch)
+        if i % 2 == 1:
+            d *= 2
+            if d > 9:
+                d -= 9
+        total += d
+    return total % 10 == 0
+
+
+class Redactor:
+    """Replaces matches with [REDACTED_<KIND>] and counts what it replaced."""
+
+    def __init__(self, enabled: bool = True) -> None:
+        self.enabled = enabled
+        self.counts: Counter[str] = Counter()
+        self.by_trace: dict[str, Counter[str]] = {}
+        self._current: str | None = None
+
+    def start_trace(self, trace_id: str) -> None:
+        self._current = trace_id
+
+    def text(self, s: str, count: bool = True) -> str:
+        """Redact one string. count=False redacts text already counted elsewhere."""
+        if not self.enabled or not s:
+            return s
+        for kind, pat in _PATTERNS:
+
+            def repl(m: re.Match[str], kind: str = kind) -> str:
+                return self._replace(kind, m.group(0), count)
+
+            s = pat.sub(repl, s)
+        return s
+
+    def _replace(self, kind: str, match: str, count: bool = True) -> str:
+        if kind == "credit_card":
+            digits = re.sub(r"\D", "", match)
+            if not (13 <= len(digits) <= 19 and _luhn_ok(digits)):
+                return match
+        if not count:
+            return f"[REDACTED_{kind.upper()}]"
+        self.counts[kind] += 1
+        if self._current is not None:
+            self.by_trace.setdefault(self._current, Counter())[kind] += 1
+        return f"[REDACTED_{kind.upper()}]"
+
+    def value(self, v: Any) -> Any:
+        if isinstance(v, str):
+            return self.text(v)
+        if isinstance(v, list):
+            return [self.value(x) for x in v]
+        if isinstance(v, dict):
+            return {k: self.value(x) for k, x in v.items()}
+        return v
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "enabled": self.enabled,
+            "total": sum(self.counts.values()),
+            "by_kind": dict(sorted(self.counts.items())),
+            "traces_affected": len(self.by_trace),
+            "by_trace": {t: dict(c) for t, c in sorted(self.by_trace.items())},
+            "note": "Values are replaced with [REDACTED_<KIND>]; originals are never written.",
+        }