gonogo-eval 0.1.0__py3-none-any.whl

gonogo/__init__.py

@@ -0,0 +1,44 @@
+"""gonogo -- decide whether an agent is good enough to ship.
+
+Not an eval framework. A decision harness for pilot-scale case sets, where the
+honest answer is often "not enough evidence yet" and sometimes "don't automate
+this at all".
+"""
+
+from .cases import Case, CaseResult, Prediction
+from .decide import Decision, Verdict, decide
+from .evaluate import evaluate
+from .report import Report
+from .scoring import (
+    JudgeValidation,
+    exact,
+    fields,
+    judge,
+    judge_agreement,
+    numeric,
+    set_f1,
+    validate_judge,
+)
+from .stats import (
+    Interval,
+    OperatingPoint,
+    best_operating_point,
+    expected_calibration_error,
+    reliability_table,
+    required_n,
+    risk_coverage,
+    wilson,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Case", "CaseResult", "Prediction",
+    "Decision", "Verdict", "decide",
+    "evaluate", "Report",
+    "exact", "fields", "judge", "judge_agreement", "numeric", "set_f1",
+    "JudgeValidation", "validate_judge",
+    "Interval", "OperatingPoint", "best_operating_point",
+    "expected_calibration_error", "reliability_table", "required_n",
+    "risk_coverage", "wilson",
+]

gonogo/cases.py

@@ -0,0 +1,103 @@
+"""Real cases, loaded from disk.
+
+A case set is the pilot's ground truth. It is deliberately a plain JSONL file
+so a domain expert can build and audit one in a spreadsheet without touching
+Python.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterator
+
+
+@dataclass
+class Case:
+    """One real case: an input, the expected output, and whatever context helps."""
+
+    input: Any
+    expected: Any
+    id: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_jsonl(cls, path: str | Path) -> list["Case"]:
+        """Load cases from a JSONL file with `input` and `expected` keys.
+
+        Any other keys land in `metadata`, so you can slice a report by
+        customer, document type, or difficulty later.
+        """
+        path = Path(path)
+        cases: list[Case] = []
+        with path.open(encoding="utf-8") as fh:
+            for lineno, line in enumerate(fh, 1):
+                line = line.strip()
+                if not line or line.startswith("//"):
+                    continue
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError as exc:
+                    raise ValueError(f"{path}:{lineno}: invalid JSON: {exc}") from exc
+                if not isinstance(row, dict):
+                    raise ValueError(f"{path}:{lineno}: each line must be a JSON object")
+                for required in ("input", "expected"):
+                    if required not in row:
+                        raise ValueError(f"{path}:{lineno}: missing required key {required!r}")
+                cases.append(cls(
+                    input=row["input"],
+                    expected=row["expected"],
+                    id=row.get("id") or f"case-{lineno}",
+                    metadata={k: v for k, v in row.items() if k not in ("input", "expected", "id")},
+                ))
+        if not cases:
+            raise ValueError(f"{path}: no cases found")
+        return cases
+
+    @staticmethod
+    def to_jsonl(cases: list["Case"], path: str | Path) -> None:
+        with Path(path).open("w", encoding="utf-8") as fh:
+            for c in cases:
+                row = {"id": c.id, "input": c.input, "expected": c.expected, **c.metadata}
+                fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+@dataclass
+class Prediction:
+    """What the agent returned for one case.
+
+    `confidence` is optional but it is what unlocks selective automation; without
+    it the harness can only report an overall pass rate.
+    """
+
+    output: Any
+    confidence: float | None = None
+    abstained: bool = False
+    error: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.confidence is not None and not 0.0 <= self.confidence <= 1.0:
+            raise ValueError(f"confidence must be in [0, 1], got {self.confidence}")
+
+
+@dataclass
+class CaseResult:
+    """A scored case: what was expected, what came back, and whether it passed."""
+
+    case: Case
+    prediction: Prediction
+    passed: bool
+    score: float
+    detail: str = ""
+
+    @property
+    def confidence(self) -> float:
+        # A case with no stated confidence is treated as fully confident, which
+        # keeps it in every coverage bucket rather than silently excusing it.
+        return 1.0 if self.prediction.confidence is None else self.prediction.confidence
+
+
+def iter_batches(cases: list[Case], size: int) -> Iterator[list[Case]]:
+    for i in range(0, len(cases), size):
+        yield cases[i:i + size]

gonogo/decide.py

@@ -0,0 +1,165 @@
+"""The verdict layer.
+
+Every other eval tool stops at a number. This module turns the number into a
+deployment decision, and refuses to when the evidence is too thin.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+from .stats import (
+    Interval,
+    OperatingPoint,
+    best_operating_point,
+    expected_calibration_error,
+    required_n,
+    wilson,
+)
+
+
+class Verdict(str, Enum):
+    AUTOMATE = "AUTOMATE"
+    AUTOMATE_WITH_REVIEW = "AUTOMATE WITH REVIEW"
+    ASSIST_ONLY = "ASSIST ONLY"
+    DO_NOT_AUTOMATE = "DO NOT AUTOMATE"
+    INSUFFICIENT_EVIDENCE = "INSUFFICIENT EVIDENCE"
+
+
+# Above this, miscalibrated confidence makes an abstention threshold meaningless.
+ECE_UNUSABLE = 0.15
+# An operating point covering less than this isn't worth the plumbing.
+MIN_USEFUL_COVERAGE = 0.25
+# Below this pass rate, tuning won't save it.
+ASSIST_FLOOR = 0.50
+
+
+@dataclass
+class Decision:
+    verdict: Verdict
+    reason: str
+    pass_rate: Interval
+    target: float
+    operating_point: OperatingPoint | None = None
+    calibration_error: float = float("nan")
+    needed_n: int | None = None
+    notes: list[str] = field(default_factory=list)
+
+    @property
+    def can_automate(self) -> bool:
+        return self.verdict in (Verdict.AUTOMATE, Verdict.AUTOMATE_WITH_REVIEW)
+
+
+def decide(
+    results: list[tuple[float, bool]],
+    target: float = 0.95,
+    level: float = 0.95,
+    min_coverage: float = MIN_USEFUL_COVERAGE,
+) -> Decision:
+    """Turn per-case (confidence, passed) results into a deployment decision.
+
+    The ordering is deliberate. We ask "is the whole task good enough?" first,
+    then "is some confident subset good enough?", and only then do we consider
+    whether the sample is simply too small to say. A tool that reports
+    INSUFFICIENT EVIDENCE when it should report DO NOT AUTOMATE is just as
+    dishonest as one that reports a bare point estimate.
+    """
+    if not 0.0 < target < 1.0:
+        raise ValueError(f"target must be in (0, 1), got {target}")
+
+    n = len(results)
+    if n == 0:
+        return Decision(
+            verdict=Verdict.INSUFFICIENT_EVIDENCE,
+            reason="no cases were evaluated",
+            pass_rate=wilson(0, 0, level),
+            target=target,
+        )
+
+    passed = sum(1 for _, p in results if p)
+    rate = wilson(passed, n, level)
+    ece = expected_calibration_error(results)
+    notes: list[str] = []
+
+    has_confidence = len({c for c, _ in results}) > 1
+    if not has_confidence:
+        notes.append(
+            "Every case reported the same confidence, so no abstention threshold "
+            "can be derived. Emit a real per-case confidence to unlock selective automation."
+        )
+
+    # 1. The whole task clears the bar on its own.
+    if rate.low >= target:
+        return Decision(
+            verdict=Verdict.AUTOMATE,
+            reason=(f"pass rate {rate} clears the {target:.0%} target across all cases "
+                    f"at the {level:.0%} level"),
+            pass_rate=rate, target=target, calibration_error=ece, notes=notes,
+        )
+
+    # 2. A confident subset clears the bar, with the remainder going to a human.
+    #
+    # Note that a high calibration error does NOT disqualify a threshold. The
+    # precision at each cut point is measured directly from the results and
+    # carries its own interval, so it stands whether or not the confidence
+    # number is on a meaningful scale. Miscalibration only means the threshold
+    # value is a cut point rather than a probability -- worth saying out loud,
+    # not worth throwing away a working operating point over.
+    if has_confidence:
+        if ece > ECE_UNUSABLE:
+            notes.append(
+                f"Calibration error {ece:.2f} exceeds {ECE_UNUSABLE:.2f}: the confidence score "
+                f"ranks cases usefully but its scale is not a probability. Treat any threshold "
+                f"below as an opaque cut point, and recalibrate before reading it as a percentage."
+            )
+        point = best_operating_point(results, target, min_coverage, level)
+        if point is not None:
+            notes.append(
+                f"The {point.threshold:.2f} threshold was chosen by searching this same case "
+                f"set, so its precision is optimistically biased. Re-measure it on fresh cases "
+                f"before relying on it."
+            )
+            return Decision(
+                verdict=Verdict.AUTOMATE_WITH_REVIEW,
+                reason=(f"overall pass rate {rate} misses the {target:.0%} target, but "
+                        f"abstaining below confidence {point.threshold:.2f} reaches "
+                        f"{point.precision.point:.1%} precision on {point.coverage:.0%} of cases"),
+                pass_rate=rate, target=target, operating_point=point,
+                calibration_error=ece, notes=notes,
+            )
+        near = best_operating_point(results, target, 0.0, level)
+        if near is not None:
+            notes.append(
+                f"A threshold of {near.threshold:.2f} would hit the target but only covers "
+                f"{near.coverage:.0%} of cases, below the {min_coverage:.0%} floor."
+            )
+
+    # 3. The point estimate looks good but the sample can't support the claim.
+    if rate.point >= target:
+        needed = required_n(rate.point, target, level)
+        return Decision(
+            verdict=Verdict.INSUFFICIENT_EVIDENCE,
+            reason=(f"observed {rate.point:.1%} on {n} cases is above the {target:.0%} target, "
+                    f"but the {level:.0%} interval reaches down to {rate.low:.1%}"),
+            pass_rate=rate, target=target, calibration_error=ece, needed_n=needed,
+            notes=notes + ([f"At this rate, about {needed} cases would be needed to claim "
+                            f"{target:.0%}; you have {n}."] if needed else []),
+        )
+
+    # 4. Too weak to automate, but a human-in-the-loop assist may still pay off.
+    if rate.point >= ASSIST_FLOOR:
+        return Decision(
+            verdict=Verdict.ASSIST_ONLY,
+            reason=(f"pass rate {rate} is well short of the {target:.0%} target and no "
+                    f"confident subset reaches it; useful as a draft-generator, not as "
+                    f"an unattended step"),
+            pass_rate=rate, target=target, calibration_error=ece, notes=notes,
+        )
+
+    return Decision(
+        verdict=Verdict.DO_NOT_AUTOMATE,
+        reason=(f"pass rate {rate} is below {ASSIST_FLOOR:.0%}; this workflow is not a fit "
+                f"for automation as currently scoped"),
+        pass_rate=rate, target=target, calibration_error=ece, notes=notes,
+    )

gonogo/evaluate.py

@@ -0,0 +1,83 @@
+"""The runner: point it at your agent and a case file, get a decision."""
+
+from __future__ import annotations
+
+import concurrent.futures as futures
+from typing import Any, Callable
+
+from .cases import Case, CaseResult, Prediction
+from .decide import MIN_USEFUL_COVERAGE, decide
+from .report import Report
+from .scoring import Scorer, exact
+
+# An agent is any callable from a Case to its answer. Returning a bare value is
+# fine; return a Prediction when you can also report confidence.
+Agent = Callable[[Case], Any]
+
+
+def evaluate(
+    agent: Agent,
+    cases: list[Case],
+    scorer: Scorer | None = None,
+    task: str = "task",
+    target: float = 0.95,
+    level: float = 0.95,
+    min_coverage: float = MIN_USEFUL_COVERAGE,
+    workers: int = 1,
+) -> Report:
+    """Run `agent` over `cases`, score it, and decide whether it can ship.
+
+    An agent that raises is recorded as a failed case rather than aborting the
+    run: a crash on 3 of 60 cases is a result about reliability, not a reason to
+    lose the other 57.
+    """
+    if not cases:
+        raise ValueError("no cases to evaluate")
+    scorer = scorer or exact()
+
+    if workers > 1:
+        with futures.ThreadPoolExecutor(max_workers=workers) as pool:
+            predictions = list(pool.map(lambda c: _run_one(agent, c), cases))
+    else:
+        predictions = [_run_one(agent, c) for c in cases]
+
+    results: list[CaseResult] = []
+    for case, pred in zip(cases, predictions):
+        if pred.error is not None:
+            results.append(CaseResult(case, pred, passed=False, score=0.0,
+                                      detail=f"agent error: {pred.error}"))
+            continue
+        if pred.abstained:
+            # An explicit abstention is not a correct answer, but it is honest.
+            # It lands at zero confidence so thresholding defers it first.
+            results.append(CaseResult(
+                case, Prediction(pred.output, confidence=0.0, abstained=True),
+                passed=False, score=0.0, detail="agent abstained",
+            ))
+            continue
+        try:
+            passed, score, detail = scorer(pred.output, case.expected)
+        except Exception as exc:  # a broken scorer shouldn't look like a broken agent
+            results.append(CaseResult(case, pred, passed=False, score=0.0,
+                                      detail=f"scorer error: {exc!r}"))
+            continue
+        results.append(CaseResult(case, pred, passed=passed, score=score, detail=detail))
+
+    decision = decide(
+        [(r.confidence, r.passed) for r in results],
+        target=target, level=level, min_coverage=min_coverage,
+    )
+    return Report(task=task, results=results, decision=decision,
+                  metadata={"scorer": getattr(scorer, "__qualname__", repr(scorer))})
+
+
+def _run_one(agent: Agent, case: Case) -> Prediction:
+    try:
+        raw = agent(case)
+    except Exception as exc:
+        return Prediction(output=None, error=f"{type(exc).__name__}: {exc}")
+    if isinstance(raw, Prediction):
+        return raw
+    if isinstance(raw, tuple) and len(raw) == 2 and isinstance(raw[1], (int, float)):
+        return Prediction(output=raw[0], confidence=float(raw[1]))
+    return Prediction(output=raw)