eval-builder 0.1.0__py3-none-any.whl

eval_builder/draft.py

@@ -0,0 +1,271 @@
+"""Write case and rubric skeletons for the agent to fill in, and validate them.
+
+The tool never writes expected behavior itself. It copies the observed input and
+output from the selected traces and leaves TODO markers that validation refuses
+to accept on cases marked `ready`.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .ingest import load_traces
+from .io import read_json, read_yaml, write_yaml
+from .workspace import Workspace
+
+STATUSES = ("draft", "ready", "dropped")
+MODES = ("pointwise", "pairwise")
+TODO = "TODO"
+
+CASES_HEADER = """eval-builder case file. Fill in expected_behavior and criteria with the user,
+then set status: ready (or dropped). Validation rejects ready cases that still
+contain TODO. Criteria ids must exist in rubric.yaml.
+Optional per case: reference_output (an ideal answer) and compare_output (a second
+output for pairwise judging, for example from a new prompt version)."""
+
+RUBRIC_HEADER = """eval-builder rubric. Define the criteria your cases reference and the judges
+you plan to check. A judge prompt may use {input}, {context} (earlier turns), {output},
+{answer_a}, {answer_b}, {expected_behavior} and {criteria}; judge-plan renders it per
+request."""
+
+UPDATABLE = (
+    "expected_behavior",
+    "criteria",
+    "reference_output",
+    "compare_output",
+    "status",
+    "notes",
+    "tags",
+)
+
+
+def _rubric_skeleton() -> dict[str, Any]:
+    return {
+        "version": 1,
+        "criteria": [
+            {
+                "id": "TODO-criterion-id",
+                "description": "TODO: what must be true of a good answer",
+                "scale": "pass_fail",
+            },
+        ],
+        "judges": [
+            {
+                "id": "TODO-judge-id",
+                "mode": "pointwise",
+                "criteria": ["TODO-criterion-id"],
+                "labels": ["pass", "fail"],
+                "prompt": "TODO: the exact judge prompt. Use {input}, {output}, "
+                "{expected_behavior}, {criteria}.",
+            },
+        ],
+    }
+
+
+def draft(
+    workspace: str | Path, suite: str = "eval-builder suite", force: bool = False
+) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    if not ws.selection.exists():
+        raise FileNotFoundError(f"{ws.selection} not found; run `eval-builder select` first")
+    selection = read_json(ws.selection)
+    traces = {t.id: t for t in load_traces(ws.root)}
+    existing: dict[str, Any] = {}
+    if ws.cases.exists() and not force:
+        existing = read_yaml(ws.cases) or {}
+    cases = list(existing.get("cases") or [])
+    known = {c.get("trace_id") for c in cases}
+    next_num = len(cases) + 1
+    added = 0
+    for pick in selection["selected"]:
+        tid = pick["trace_id"]
+        if tid in known:
+            continue
+        t = traces.get(tid)
+        if t is None:
+            continue
+        case: dict[str, Any] = {
+            "id": f"case-{next_num:03d}",
+            "trace_id": tid,
+            "status": "draft",
+            "selected_because": pick["reasons"],
+            "input": t.input,
+        }
+        ctx = t.prior_turns()
+        if ctx:
+            case["context"] = ctx
+        case["observed_output"] = t.output
+        case["observed_failure"] = bool(pick.get("failure"))
+        case["expected_behavior"] = f"{TODO}: one or two sentences on what a good answer does"
+        case["criteria"] = ["TODO-criterion-id"]
+        case["reference_output"] = None
+        tags = []
+        if t.route:
+            tags.append(f"route:{t.route}")
+        if t.model:
+            tags.append(f"model:{t.model}")
+        tags += [f"tool:{x}" for x in t.tools]
+        if t.error:
+            tags.append("error")
+        if t.feedback:
+            tags.append(f"feedback:{t.feedback}")
+        tags.append(f"cluster:{pick['cluster']}")
+        case["tags"] = tags
+        cases.append(case)
+        known.add(tid)
+        next_num += 1
+        added += 1
+    doc = {"suite": existing.get("suite", suite), "version": 1, "cases": cases}
+    write_yaml(ws.cases, doc, CASES_HEADER)
+    rubric_created = False
+    if not ws.rubric.exists() or force:
+        write_yaml(ws.rubric, _rubric_skeleton(), RUBRIC_HEADER)
+        rubric_created = True
+    return {
+        "cases_file": str(ws.cases),
+        "rubric_file": str(ws.rubric),
+        "cases_total": len(cases),
+        "cases_added": added,
+        "rubric_created": rubric_created,
+        "next": "fill expected_behavior and criteria per case, define criteria and judges "
+        "in rubric.yaml, then run `eval-builder validate`",
+    }
+
+
+def load_cases(workspace: str | Path) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    if not ws.cases.exists():
+        raise FileNotFoundError(f"{ws.cases} not found; run `eval-builder draft` first")
+    return read_yaml(ws.cases) or {"cases": []}
+
+
+def load_rubric(workspace: str | Path) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    if not ws.rubric.exists():
+        raise FileNotFoundError(f"{ws.rubric} not found; run `eval-builder draft` first")
+    return read_yaml(ws.rubric) or {}
+
+
+def _has_todo(v: Any) -> bool:
+    if isinstance(v, str):
+        return TODO in v
+    if isinstance(v, list):
+        return any(_has_todo(x) for x in v)
+    if isinstance(v, dict):
+        return any(_has_todo(x) for x in v.values())
+    return False
+
+
+def validate_rubric(rubric: dict[str, Any]) -> list[dict[str, str]]:
+    errors: list[dict[str, str]] = []
+
+    def err(where: str, problem: str) -> None:
+        errors.append({"where": where, "problem": problem})
+
+    crit = rubric.get("criteria") or []
+    if not crit:
+        err("rubric.criteria", "no criteria defined")
+    seen: set[str] = set()
+    for i, c in enumerate(crit):
+        cid = str(c.get("id", ""))
+        if not cid or _has_todo(cid):
+            err(f"rubric.criteria[{i}].id", "missing or still TODO")
+        if cid in seen:
+            err(f"rubric.criteria[{i}].id", f"duplicate id {cid}")
+        seen.add(cid)
+        if not str(c.get("description", "")).strip() or _has_todo(c.get("description")):
+            err(f"rubric.criteria[{i}].description", "missing or still TODO")
+    judge_ids: set[str] = set()
+    for i, j in enumerate(rubric.get("judges") or []):
+        jid = str(j.get("id", ""))
+        if not jid or _has_todo(jid):
+            err(f"rubric.judges[{i}].id", "missing or still TODO")
+        if jid in judge_ids:
+            err(f"rubric.judges[{i}].id", f"duplicate id {jid}")
+        judge_ids.add(jid)
+        if j.get("mode", "pointwise") not in MODES:
+            err(f"rubric.judges[{i}].mode", f"must be one of {', '.join(MODES)}")
+        if not j.get("labels"):
+            err(f"rubric.judges[{i}].labels", "list the verdict labels the judge may return")
+        if _has_todo(j.get("prompt")):
+            err(
+                f"rubric.judges[{i}].prompt",
+                "still TODO (remove the key if you run judges with your own prompt elsewhere)",
+            )
+        for cid in j.get("criteria") or []:
+            if cid not in seen:
+                err(f"rubric.judges[{i}].criteria", f"unknown criterion {cid}")
+    return errors
+
+
+def validate(workspace: str | Path) -> dict[str, Any]:
+    doc = load_cases(workspace)
+    rubric = load_rubric(workspace)
+    errors = validate_rubric(rubric)
+    crit_ids = {str(c.get("id")) for c in rubric.get("criteria") or []}
+    counts = dict.fromkeys(STATUSES, 0)
+    ids: set[str] = set()
+    for i, c in enumerate(doc.get("cases") or []):
+        cid = str(c.get("id", f"#{i}"))
+        where = f"cases[{cid}]"
+        if cid in ids:
+            errors.append({"where": where, "problem": "duplicate case id"})
+        ids.add(cid)
+        status = c.get("status", "draft")
+        if status not in STATUSES:
+            errors.append({"where": where, "problem": f"status must be one of {STATUSES}"})
+            continue
+        counts[status] += 1
+        if status != "ready":
+            continue
+        if not str(c.get("input", "")).strip():
+            errors.append({"where": where, "problem": "empty input"})
+        eb = c.get("expected_behavior")
+        if not isinstance(eb, str) or not eb.strip() or _has_todo(eb):
+            errors.append(
+                {"where": f"{where}.expected_behavior", "problem": "missing or still TODO"}
+            )
+        crit = c.get("criteria") or []
+        if not crit:
+            errors.append({"where": f"{where}.criteria", "problem": "no criteria"})
+        for k in crit:
+            if k not in crit_ids:
+                errors.append({"where": f"{where}.criteria", "problem": f"unknown criterion {k}"})
+        if _has_todo(c.get("reference_output")):
+            errors.append({"where": f"{where}.reference_output", "problem": "still TODO"})
+    if counts["ready"] == 0:
+        errors.append({"where": "cases", "problem": "no cases have status: ready"})
+    warnings = []
+    if counts["draft"]:
+        warnings.append(f"{counts['draft']} case(s) still draft; they are not exported")
+    return {"valid": not errors, "counts": counts, "errors": errors, "warnings": warnings}
+
+
+def update_case(workspace: str | Path, case_id: str, **fields: Any) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    doc = load_cases(ws.root)
+    bad = [k for k in fields if k not in UPDATABLE]
+    if bad:
+        raise ValueError(f"cannot update {bad}; allowed: {', '.join(UPDATABLE)}")
+    if "status" in fields and fields["status"] not in STATUSES:
+        raise ValueError(f"status must be one of {STATUSES}")
+    for c in doc.get("cases") or []:
+        if c.get("id") == case_id:
+            for k, v in fields.items():
+                if v is not None:
+                    c[k] = v
+            write_yaml(ws.cases, doc, CASES_HEADER)
+            return c
+    raise KeyError(f"no case with id {case_id}")
+
+
+def write_rubric(workspace: str | Path, rubric: dict[str, Any]) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    rubric = {"version": 1, **rubric}
+    write_yaml(ws.rubric, rubric, RUBRIC_HEADER)
+    return {"rubric_file": str(ws.rubric), "errors": validate_rubric(rubric)}
+
+
+def ready_cases(workspace: str | Path) -> list[dict[str, Any]]:
+    return [c for c in load_cases(workspace).get("cases") or [] if c.get("status") == "ready"]

eval_builder/export.py

@@ -0,0 +1,284 @@
+"""Export ready cases to the eval tools people already use.
+
+eval-builder is not an eval runner. It writes files for promptfoo, DeepEval and
+Inspect AI (and plain JSONL) so the suite runs where your team already works.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from .draft import load_cases, load_rubric, validate
+from .io import read_json, sha256_file, write_json, write_jsonl, write_yaml
+from .workspace import Workspace
+
+FORMATS = ("promptfoo", "deepeval", "inspect", "jsonl")
+
+
+def _criteria_text(case: dict[str, Any], crit: dict[str, str]) -> str:
+    return "\n".join(f"- {c}: {crit.get(c, '')}" for c in case.get("criteria") or [])
+
+
+def _metadata(case: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "case_id": case["id"],
+        "trace_id": case.get("trace_id"),
+        "tags": case.get("tags") or [],
+        "criteria": case.get("criteria") or [],
+    }
+
+
+def _judge_summary(ws: Workspace) -> dict[str, Any] | None:
+    if not ws.judge_check.exists():
+        return None
+    jc = read_json(ws.judge_check)
+    return {
+        "trustworthy": jc.get("trustworthy", []),
+        "verdicts": {s["judge"]: s["verdict"] for s in jc.get("summary", [])},
+        "thresholds": jc.get("thresholds"),
+    }
+
+
+def _export_jsonl(cases: list[dict[str, Any]], crit: dict[str, str], out: Path) -> list[Path]:
+    rows = []
+    for c in cases:
+        rows.append(
+            {
+                "id": c["id"],
+                "input": c.get("input"),
+                "context": c.get("context") or [],
+                "expected_behavior": c.get("expected_behavior"),
+                "criteria": [
+                    {"id": k, "description": crit.get(k, "")} for k in c.get("criteria") or []
+                ],
+                "reference_output": c.get("reference_output"),
+                "observed_output": c.get("observed_output"),
+                "metadata": _metadata(c),
+            }
+        )
+    path = out / "cases.jsonl"
+    write_jsonl(path, rows)
+    return [path]
+
+
+PROMPTFOO_HEADER = """Generated by eval-builder. Replace the `echo` provider with your app's
+provider (https://www.promptfoo.dev/docs/providers/). `echo` only returns the prompt, so a run
+with it checks the plumbing, not your app. llm-rubric assertions need a grader model
+configured in promptfoo (defaultTest.options.provider)."""
+
+
+def _export_promptfoo(
+    cases: list[dict[str, Any]], crit: dict[str, str], suite: str, out: Path
+) -> list[Path]:
+    tests = []
+    for c in cases:
+        variables: dict[str, Any] = {"input": c.get("input")}
+        if c.get("context"):
+            variables["context"] = json.dumps(c["context"], ensure_ascii=False)
+        rubric = f"{c.get('expected_behavior', '').strip()}\n\nCriteria:\n{_criteria_text(c, crit)}"
+        test: dict[str, Any] = {
+            "description": f"{c['id']}: {str(c.get('input', ''))[:60].strip()}",
+            "vars": variables,
+            "assert": [{"type": "llm-rubric", "value": rubric}],
+            "metadata": {
+                "case_id": c["id"],
+                "trace_id": c.get("trace_id"),
+                "criteria": ",".join(c.get("criteria") or []),
+            },
+        }
+        tests.append(test)
+    config = {"description": suite, "prompts": ["{{input}}"], "providers": ["echo"], "tests": tests}
+    path = out / "promptfooconfig.yaml"
+    write_yaml(path, config, PROMPTFOO_HEADER)
+    return [path]
+
+
+DEEPEVAL_TEST = '''"""Generated by eval-builder. Run with: deepeval test run test_eval_builder.py
+
+Fill in `your_app` to call the system under test. Set EVAL_BUILDER_USE_OBSERVED=1 to
+score the outputs that were logged instead (a regression check on past behavior).
+GEval needs a judge model configured in DeepEval.
+"""
+
+import json
+import os
+from pathlib import Path
+
+import pytest
+from deepeval import assert_test
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCase
+
+try:  # DeepEval 4.x
+    from deepeval.test_case import SingleTurnParams as Params
+except ImportError:  # older DeepEval
+    from deepeval.test_case import LLMTestCaseParams as Params
+
+CASES = json.loads((Path(__file__).parent / "dataset.json").read_text(encoding="utf-8"))
+
+
+def your_app(input: str) -> str:
+    raise NotImplementedError("call the app under test here")
+
+
+def make_metric(criteria: str) -> GEval:
+    return GEval(
+        name="expected behavior",
+        criteria=(
+            "Judge whether the actual output does what the expected output describes. "
+            "The expected output is a description of good behavior, not an exact answer. "
+            + criteria
+        ),
+        evaluation_params=[
+            Params.INPUT,
+            Params.ACTUAL_OUTPUT,
+            Params.EXPECTED_OUTPUT,
+        ],
+    )
+
+
+@pytest.mark.parametrize("case", CASES, ids=[c["name"] for c in CASES])
+def test_case(case):
+    if os.environ.get("EVAL_BUILDER_USE_OBSERVED") == "1":
+        actual = case["actual_output"]
+    else:
+        actual = your_app(case["input"])
+    test_case = LLMTestCase(
+        input=case["input"],
+        actual_output=actual,
+        expected_output=case["expected_output"],
+        context=case.get("context") or None,
+    )
+    assert_test(test_case, [make_metric(case["additional_metadata"]["criteria_text"])])
+'''
+
+
+def _export_deepeval(cases: list[dict[str, Any]], crit: dict[str, str], out: Path) -> list[Path]:
+    rows = []
+    for c in cases:
+        ctx = [f"{m['role']}: {m['content']}" for m in c.get("context") or []]
+        rows.append(
+            {
+                "name": c["id"],
+                "input": c.get("input"),
+                "expected_output": c.get("expected_behavior"),
+                "actual_output": c.get("observed_output"),
+                "context": ctx or None,
+                "additional_metadata": {**_metadata(c), "criteria_text": _criteria_text(c, crit)},
+            }
+        )
+    data = out / "dataset.json"
+    write_json(data, rows)
+    test = out / "test_eval_builder.py"
+    test.write_text(DEEPEVAL_TEST, encoding="utf-8")
+    return [data, test]
+
+
+INSPECT_TASK = '''"""Generated by eval-builder.
+
+Run with: inspect eval task.py --model <provider/model>
+
+`target` holds the expected behavior written for each case; model_graded_qa asks a
+grader model whether the answer meets it.
+"""
+
+from pathlib import Path
+
+from inspect_ai import Task, task
+from inspect_ai.dataset import json_dataset
+from inspect_ai.scorer import model_graded_qa
+from inspect_ai.solver import generate
+
+
+@task
+def eval_builder_suite():
+    return Task(
+        dataset=json_dataset(str(Path(__file__).parent / "dataset.jsonl")),
+        solver=generate(),
+        scorer=model_graded_qa(),
+    )
+'''
+
+
+def _export_inspect(cases: list[dict[str, Any]], crit: dict[str, str], out: Path) -> list[Path]:
+    rows = []
+    for c in cases:
+        ctx = c.get("context") or []
+        inp: Any = c.get("input")
+        if ctx:
+            inp = [
+                {"role": m["role"], "content": m["content"]}
+                for m in ctx
+                if m.get("role") in ("system", "user", "assistant")
+            ]
+            inp.append({"role": "user", "content": c.get("input")})
+        target = c.get("expected_behavior", "").strip()
+        criteria = _criteria_text(c, crit)
+        if criteria:
+            target = f"{target}\n\nCriteria:\n{criteria}"
+        rows.append(
+            {
+                "id": c["id"],
+                "input": inp,
+                "target": target,
+                "metadata": {**_metadata(c), "observed_output": c.get("observed_output")},
+            }
+        )
+    data = out / "dataset.jsonl"
+    write_jsonl(data, rows)
+    task_file = out / "task.py"
+    task_file.write_text(INSPECT_TASK, encoding="utf-8")
+    return [data, task_file]
+
+
+def export(workspace: str | Path, formats: list[str] | None = None) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    formats = formats or list(FORMATS)
+    bad = [f for f in formats if f not in FORMATS]
+    if bad:
+        raise ValueError(f"unknown export format(s) {bad}; expected {', '.join(FORMATS)}")
+    check = validate(ws.root)
+    if not check["valid"]:
+        return {
+            "exported": False,
+            "validation": check,
+            "next": "fix the validation errors, then export again",
+        }
+    doc = load_cases(ws.root)
+    rubric = load_rubric(ws.root)
+    crit = {str(c["id"]): str(c.get("description", "")) for c in rubric.get("criteria") or []}
+    cases = [c for c in doc.get("cases") or [] if c.get("status") == "ready"]
+    suite = str(doc.get("suite", "eval-builder suite"))
+    files: list[Path] = []
+    for f in formats:
+        out = ws.exports / f
+        out.mkdir(parents=True, exist_ok=True)
+        if f == "jsonl":
+            files += _export_jsonl(cases, crit, out)
+        elif f == "promptfoo":
+            files += _export_promptfoo(cases, crit, suite, out)
+        elif f == "deepeval":
+            files += _export_deepeval(cases, crit, out)
+        elif f == "inspect":
+            files += _export_inspect(cases, crit, out)
+    judges = _judge_summary(ws)
+    trusted_prompts = {}
+    if judges:
+        for j in rubric.get("judges") or []:
+            if j.get("id") in judges["trustworthy"] and j.get("prompt"):
+                trusted_prompts[j["id"]] = {
+                    "mode": j.get("mode"),
+                    "labels": j.get("labels"),
+                    "prompt": j["prompt"],
+                }
+    manifest = {
+        "cases": len(cases),
+        "formats": formats,
+        "files": [{"path": str(p.relative_to(ws.root)), "sha256": sha256_file(p)} for p in files],
+        "judge_check": judges,
+        "trusted_judge_prompts": trusted_prompts,
+    }
+    write_json(ws.exports / "manifest.json", manifest)
+    return {"exported": True, **manifest}