eval-builder 0.1.0__py3-none-any.whl

eval_builder/report.py

@@ -0,0 +1,263 @@
+"""Markdown and JSON report over whatever steps have run in a workspace."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .draft import load_cases
+from .io import read_json, write_json
+from .workspace import Workspace
+
+STATIC_LIMITS = [
+    "Selection is lexical (TF-IDF over the user input). Two inputs that mean the same thing "
+    "in different words can land in different clusters, and near-duplicate detection only "
+    "catches close textual matches.",
+    "Redaction is pattern-based (emails, common API key formats, Luhn-valid card numbers, "
+    "US-style phone numbers, SSN-shaped numbers). Names, addresses and free-form secrets are "
+    "not detected. Review traces before sharing them.",
+    "eval-builder does not write expected behavior. Cases are only as good as what the agent "
+    "and user put in cases.yaml.",
+    "Judge verdicts depend on the thresholds shown and on how many cases, trials and human "
+    "labels were provided. Small samples give wide intervals; read the intervals, not just "
+    "the point estimates.",
+    "The bias probes cover answer order and irrelevant padding only. They do not detect "
+    "self-preference, style bias or rubric misreadings.",
+]
+
+
+_DEDUPE_LABEL = {"input": "the user turns", "input+output": "the user turns plus the output"}
+
+
+def _pct(x: float | None) -> str:
+    return "n/a" if x is None else f"{x:.0%}"
+
+
+def _ci(r: dict[str, Any] | None) -> str:
+    if not r or r.get("rate") is None:
+        return "n/a"
+    lo, hi = r["ci95"]
+    return f"{r['rate']:.0%} [{lo:.0%}, {hi:.0%}] (n={r['n']})"
+
+
+def _md_escape(s: Any) -> str:
+    return str(s).replace("|", "\\|").replace("\n", " ")
+
+
+def build_report(workspace: str | Path, title: str | None = None) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    data: dict[str, Any] = {
+        "generated_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "eval_builder_version": __version__,
+        "workspace": str(ws.root),
+    }
+    if ws.ingest_report.exists():
+        data["ingest"] = read_json(ws.ingest_report)
+    if ws.selection.exists():
+        data["selection"] = read_json(ws.selection)
+    if ws.cases.exists():
+        cases = load_cases(ws.root).get("cases") or []
+        data["cases"] = {
+            "total": len(cases),
+            "by_status": {
+                s: sum(1 for c in cases if c.get("status") == s)
+                for s in ("draft", "ready", "dropped")
+            },
+        }
+    if ws.judge_check.exists():
+        data["judge_check"] = read_json(ws.judge_check)
+    manifest = ws.exports / "manifest.json"
+    if manifest.exists():
+        data["exports"] = read_json(manifest)
+    run_log = ws.root / "judge_run_log.json"
+    if run_log.exists():
+        data["judge_run"] = read_json(run_log)
+    data["limits"] = list(STATIC_LIMITS) + _dynamic_limits(data)
+    write_json(ws.report_json, data)
+    ws.report_md.write_text(render_markdown(data, title), encoding="utf-8")
+    return {
+        "report_md": str(ws.report_md),
+        "report_json": str(ws.report_json),
+        "sections": [
+            k for k in ("ingest", "selection", "cases", "judge_check", "exports") if k in data
+        ],
+    }
+
+
+def _dynamic_limits(data: dict[str, Any]) -> list[str]:
+    out = []
+    jc = data.get("judge_check")
+    if jc and not jc.get("labels_file"):
+        out.append("No human labels were provided, so no judge can be marked trustworthy.")
+    sel = data.get("selection")
+    if sel and sel["population"]["failures_unique"] == 0:
+        out.append(
+            "No traces carried an error flag or negative feedback, so failure "
+            "oversampling had nothing to work with."
+        )
+    return out
+
+
+def render_markdown(d: dict[str, Any], title: str | None = None) -> str:
+    L: list[str] = [f"# {title or 'eval-builder report'}", ""]
+    L.append(f"Generated {d['generated_at']} by eval-builder {d['eval_builder_version']}.")
+    L.append("")
+    ing = d.get("ingest")
+    if ing:
+        L += [
+            "## Sources",
+            "",
+            "| file | format | sha256 | records | traces | skipped |",
+            "|---|---|---|---|---|---|",
+        ]
+        for s in ing["sources"]:
+            skipped = ", ".join(f"{k}: {v}" for k, v in s["skipped"].items()) or "0"
+            L.append(
+                f"| {_md_escape(Path(s['path']).name)} | {s['format']} | "
+                f"`{s['sha256'][:16]}` | {s['records']} | {s['traces']} | "
+                f"{_md_escape(skipped)} |"
+            )
+        L += [
+            "",
+            f"Traces ingested: {ing['traces']} ({ing['multi_turn']} multi-turn, "
+            f"{ing['with_error']} with an error flag). Feedback: "
+            + ", ".join(f"{k} {v}" for k, v in ing["feedback"].items())
+            + ".",
+            "",
+        ]
+        r = ing["redactions"]
+        L += ["## Redactions", ""]
+        if not r["enabled"]:
+            L.append("Redaction was turned off for this run.")
+        elif r["total"] == 0:
+            L.append("Redaction was on. No matches.")
+        else:
+            L.append(
+                f"Redaction was on. {r['total']} value(s) replaced in "
+                f"{r['traces_affected']} trace(s): "
+                + ", ".join(f"{k} {v}" for k, v in r["by_kind"].items())
+                + "."
+            )
+        L.append("")
+    sel = d.get("selection")
+    if sel:
+        p = sel["population"]
+        L += [
+            "## Selection",
+            "",
+            f"{p['traces']} traces, {p['exact_duplicates_removed']} exact duplicates removed, "
+            f"{p['near_duplicates_merged']} near-duplicates merged (cosine >= "
+            f"{sel['params']['near_dup_threshold']} on "
+            f"{_DEDUPE_LABEL.get(sel['params']['dedupe_on'], sel['params']['dedupe_on'])}), "
+            f"{p['unique']} unique. Selected {sel['selected_count']} "
+            f"({sel['selected_failures']} failures; failures are {p['failures_unique']} of "
+            f"{p['unique']} unique traces). Seed {sel['params']['seed']}, "
+            f"{sel['params']['clusters']} k-means clusters.",
+            "",
+            "| cluster | traces | share | selected | top terms |",
+            "|---|---|---|---|---|",
+        ]
+        for c in sel["clusters"]:
+            L.append(
+                f"| {c['id']} | {c['size']} | {c['share']:.0%} | {c['selected']} | "
+                f"{_md_escape(', '.join(c['terms']))} |"
+            )
+        if sel.get("strata"):
+            L += ["", "| stratum | value | unique traces | selected |", "|---|---|---|---|"]
+            for f, vals in sel["strata"].items():
+                for v, cnt in vals.items():
+                    L.append(f"| {f} | {_md_escape(v)} | {cnt['population']} | {cnt['selected']} |")
+        if sel.get("strata_skipped"):
+            L.append("")
+            L.append(
+                "Not stratified: "
+                + ", ".join(f"{k} ({v})" for k, v in sel["strata_skipped"].items())
+            )
+        L += ["", "| trace | cluster | represents | why it was picked |", "|---|---|---|---|"]
+        for s in sel["selected"]:
+            L.append(
+                f"| `{s['trace_id']}` | {s['cluster']} | {s['represents']} | "
+                f"{_md_escape('; '.join(s['reasons']))} |"
+            )
+        L.append("")
+    cs = d.get("cases")
+    if cs:
+        b = cs["by_status"]
+        L += [
+            "## Cases",
+            "",
+            f"{cs['total']} cases: {b['ready']} ready, {b['draft']} draft, {b['dropped']} dropped.",
+            "",
+        ]
+    jc = d.get("judge_check")
+    if jc:
+        L += ["## Judge reliability", ""]
+        th = jc["thresholds"]
+        L.append(
+            f"Thresholds: flip rate <= {th['max_flip_rate']:.0%} (cases with >= "
+            f"{th['min_trials']} trials, >= {th['min_cases']} cases), position "
+            f"consistency >= {th['min_position_consistency']:.0%}, padding moves verdict "
+            f"toward padded answer <= {th['max_toward_padded_rate']:.0%}, Cohen's kappa "
+            f">= {th['min_kappa']} on >= {th['min_labeled']} human-labeled cases. Intervals "
+            f"are 95% (Wilson for rates, Cohen's large-sample SE for kappa). Self-agreement is "
+            "the chance one call matches the judge's own majority; majority-of-3 stable is "
+            "the chance a 3-call majority vote matches it (exact, needs >= 5 trials)."
+        )
+        L += [
+            "",
+            "| judge | mode | verdict | flip rate | self-agreement | majority-of-3 stable | "
+            "accuracy vs humans | kappa | position consistency | first-shown picked | "
+            "padding helped |",
+            "|---|---|---|---|---|---|---|---|---|---|---|",
+        ]
+        for jid, r in jc["judges"].items():
+            st, ag = r["stability"], r["human_agreement"] or {}
+            pos, verb = r["position_probe"] or {}, r["verbosity_probe"] or {}
+            kappa = "n/a"
+            if ag.get("kappa") is not None:
+                kappa = f"{ag['kappa']:.2f}"
+                if ag.get("kappa_ci95"):
+                    kappa += f" [{ag['kappa_ci95'][0]:.2f}, {ag['kappa_ci95'][1]:.2f}]"
+            L.append(
+                f"| {_md_escape(jid)} | {r['mode']} | **{r['verdict']}** | "
+                f"{_ci(st['flip_rate'])} | {_pct(st['mean_self_agreement'])} | "
+                f"{_pct(st.get('majority_of_3_stability'))} | "
+                f"{_ci(ag.get('accuracy'))} | {kappa} | {_ci(pos.get('consistency'))} | "
+                f"{_ci(pos.get('first_position_rate'))} | {_ci(verb.get('toward_padded'))} |"
+            )
+        L.append("")
+        for jid, r in jc["judges"].items():
+            L.append(
+                f"- **{jid}** ({r['verdict']}): "
+                + " ".join(s.rstrip(".") + "." for s in r["reasons"])
+            )
+        L.append("")
+    run = d.get("judge_run")
+    if run:
+        L += ["Judge calls made through the opt-in judge plugin (`judge-run`):", ""]
+        for i, rr in enumerate(run.get("runs", [run]), 1):
+            for jid, e in rr["judges"].items():
+                if not e["requests"]:
+                    continue
+                L.append(
+                    f"- run {i}, {jid}: `{_md_escape(e['command'])}`, {e['ok']} ok, "
+                    f"{e['errors']} errors, {e['seconds']} s"
+                )
+        L.append("")
+    ex = d.get("exports")
+    if ex:
+        L += [
+            "## Exports",
+            "",
+            f"{ex['cases']} ready cases exported.",
+            "",
+            "| file | sha256 |",
+            "|---|---|",
+        ]
+        for f in ex["files"]:
+            L.append(f"| {f['path']} | `{f['sha256'][:16]}` |")
+        L.append("")
+    L += ["## Limits", ""] + [f"- {x}" for x in d["limits"]] + [""]
+    return "\n".join(L)

eval_builder/schema.py

@@ -0,0 +1,242 @@
+"""The normalized trace schema every ingest format maps into."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass
+class Trace:
+    """One logged interaction, normalized.
+
+    `input` is the last user message before the final assistant reply and `output`
+    is that reply's text. `messages` keeps the whole conversation (including the
+    final reply) so multi-turn context is never lost.
+    """
+
+    id: str
+    input: str
+    output: str
+    messages: list[dict[str, str]] = field(default_factory=list)
+    system: str | None = None
+    tools: list[str] = field(default_factory=list)
+    error: bool = False
+    feedback: str | None = None  # "positive" | "negative" | "neutral" | None
+    route: str | None = None
+    model: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    source: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> Trace:
+        known = {f for f in cls.__dataclass_fields__}
+        return cls(**{k: v for k, v in d.items() if k in known})
+
+    def prior_turns(self) -> list[dict[str, str]]:
+        """Conversation before the last user message (the case input).
+
+        Tool calls and intermediate assistant steps after that message are part of the
+        app's behavior being evaluated, so they are not included.
+        """
+        last_user = max(
+            (i for i, m in enumerate(self.messages) if m.get("role") == "user"), default=0
+        )
+        return list(self.messages[:last_user])
+
+
+def text_of(content: Any) -> str:
+    """Extract plain text from the many shapes providers use for message content."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, (int, float, bool)):
+        return str(content)
+    if isinstance(content, list):
+        parts = [text_of(p) for p in content]
+        return "\n".join(p for p in parts if p)
+    if isinstance(content, dict):
+        ptype = content.get("type")
+        if ptype in ("tool_use", "tool_call", "function_call", "image", "image_url", "input_image"):
+            return ""
+        for key in ("text", "content", "value", "output_text", "input_text"):
+            if key in content:
+                return text_of(content[key])
+        return ""
+    return str(content)
+
+
+_FEEDBACK_POS = {
+    "up",
+    "thumbs_up",
+    "thumbsup",
+    "positive",
+    "good",
+    "like",
+    "liked",
+    "yes",
+    "true",
+    "+1",
+    "1",
+    "helpful",
+    "pass",
+    "correct",
+}
+_FEEDBACK_NEG = {
+    "down",
+    "thumbs_down",
+    "thumbsdown",
+    "negative",
+    "bad",
+    "dislike",
+    "disliked",
+    "no",
+    "false",
+    "-1",
+    "0",
+    "unhelpful",
+    "fail",
+    "incorrect",
+    "wrong",
+}
+
+
+def normalize_feedback(value: Any) -> str | None:
+    """Map thumbs, booleans, 0/1 and 1-5 ratings onto positive/negative/neutral."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return "positive" if value else "negative"
+    if isinstance(value, (int, float)):
+        v = float(value)
+        if v < 0:
+            return "negative"
+        if v <= 1:
+            return "negative" if v < 0.5 else "positive"
+        if v <= 5:
+            if v <= 2:
+                return "negative"
+            return "positive" if v >= 4 else "neutral"
+        return None
+    if isinstance(value, str):
+        s = value.strip().lower().replace(" ", "_")
+        if s in _FEEDBACK_POS:
+            return "positive"
+        if s in _FEEDBACK_NEG:
+            return "negative"
+        if s in ("neutral", "meh", "mixed"):
+            return "neutral"
+        try:
+            return normalize_feedback(float(s))
+        except ValueError:
+            return None
+    if isinstance(value, dict):
+        for key in ("value", "score", "rating", "label"):
+            if key in value:
+                return normalize_feedback(value[key])
+    return None
+
+
+FEEDBACK_KEYS = ("feedback", "user_feedback", "thumbs", "rating", "user_rating", "vote")
+ERROR_KEYS = ("error", "is_error", "failed", "exception")
+ROUTE_KEYS = ("route", "endpoint", "path", "feature", "workflow", "task", "intent")
+
+
+def fill_common_fields(trace: Trace) -> Trace:
+    """Derive error, feedback and route from metadata when a format did not set them."""
+    md = trace.metadata
+    if trace.feedback is None:
+        for key in FEEDBACK_KEYS:
+            if key in md:
+                trace.feedback = normalize_feedback(md[key])
+                if trace.feedback:
+                    break
+    if not trace.error:
+        for key in ERROR_KEYS:
+            v = md.get(key)
+            if v not in (None, False, "", 0, "false", "False", "none", "None"):
+                trace.error = True
+                break
+        status = str(md.get("status", "")).lower()
+        if status in ("error", "failed", "failure"):
+            trace.error = True
+    if trace.route is None:
+        for key in ROUTE_KEYS:
+            v = md.get(key)
+            if isinstance(v, (str, int)) and str(v):
+                trace.route = str(v)
+                break
+    if trace.model is None and isinstance(md.get("model"), str):
+        trace.model = md["model"]
+    return trace
+
+
+_ID_SAFE = re.compile(r"[^A-Za-z0-9_.:-]+")
+
+
+def make_id(*parts: Any, given: Any = None) -> str:
+    if given not in (None, ""):
+        cleaned = _ID_SAFE.sub("-", str(given)).strip("-")
+        if cleaned:
+            return cleaned[:80]
+    blob = json.dumps(parts, ensure_ascii=False, sort_keys=True, default=str)
+    return "t-" + hashlib.sha1(blob.encode("utf-8")).hexdigest()[:12]
+
+
+def build_trace(
+    messages: list[dict[str, Any]],
+    *,
+    fmt: str,
+    file: str,
+    index: int,
+    given_id: Any = None,
+    system: str | None = None,
+    tools: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
+    error: bool = False,
+    feedback: str | None = None,
+    route: str | None = None,
+    model: str | None = None,
+) -> Trace:
+    """Build a Trace from role/content messages (content already flattened to text)."""
+    norm: list[dict[str, str]] = []
+    sys_parts: list[str] = [system] if system else []
+    for m in messages:
+        role = str(m.get("role", "user")).lower()
+        if role in ("system", "developer"):
+            t = text_of(m.get("content"))
+            if t:
+                sys_parts.append(t)
+            continue
+        if role in ("human",):
+            role = "user"
+        if role in ("ai", "model", "bot"):
+            role = "assistant"
+        norm.append({"role": role, "content": text_of(m.get("content"))})
+    output = ""
+    if norm and norm[-1]["role"] == "assistant":
+        output = norm[-1]["content"]
+    user_msgs = [m["content"] for m in norm if m["role"] == "user"]
+    inp = user_msgs[-1] if user_msgs else ""
+    trace = Trace(
+        id=make_id(fmt, file, index, inp, output, given=given_id),
+        input=inp,
+        output=output,
+        messages=norm,
+        system="\n\n".join(sys_parts) or None,
+        tools=sorted(set(tools or [])),
+        error=error,
+        feedback=feedback,
+        route=route,
+        model=model,
+        metadata=dict(metadata or {}),
+        source={"format": fmt, "file": file, "index": index},
+    )
+    return fill_common_fields(trace)