hemp-eval 0.1.0__py3-none-any.whl

hemp/__init__.py

@@ -0,0 +1,32 @@
+"""HEMP: Help Evaluate My Prompt."""
+
+from hemp._version import __version__
+from hemp.dataset import DatasetError, load_dataset
+from hemp.metrics import MetricContext, MetricParams, MetricResult, metric
+from hemp.plan import EvalPlan, PlanError, create_plan, read_plan, write_plan
+from hemp.profile import DatasetProfile, profile_dataset, profile_rows
+from hemp.runner import RunResult, run_eval
+from hemp.store import RunDiff, Store, diff_runs
+
+__all__ = [
+    "DatasetError",
+    "DatasetProfile",
+    "EvalPlan",
+    "MetricContext",
+    "MetricParams",
+    "MetricResult",
+    "PlanError",
+    "RunDiff",
+    "RunResult",
+    "Store",
+    "__version__",
+    "create_plan",
+    "diff_runs",
+    "load_dataset",
+    "metric",
+    "profile_dataset",
+    "profile_rows",
+    "read_plan",
+    "run_eval",
+    "write_plan",
+]

hemp/_version.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("hemp-eval")
+except PackageNotFoundError:  # pragma: no cover - running from a source tree without install
+    __version__ = "0.0.0"

hemp/calibrate/__init__.py

@@ -0,0 +1,24 @@
+"""Human labeling, judge tuning, and judge-human agreement."""
+
+from hemp.calibrate.agreement import Agreement, agreement
+from hemp.calibrate.labels import Label, LabelItem, label_candidates
+from hemp.calibrate.optimize import (
+    MIN_LABELS_TO_TUNE,
+    CalibrationFailed,
+    CalibrationReport,
+    QuestionAgreement,
+    calibrate,
+)
+
+__all__ = [
+    "MIN_LABELS_TO_TUNE",
+    "Agreement",
+    "CalibrationFailed",
+    "CalibrationReport",
+    "Label",
+    "LabelItem",
+    "QuestionAgreement",
+    "agreement",
+    "calibrate",
+    "label_candidates",
+]

hemp/calibrate/agreement.py

@@ -0,0 +1,33 @@
+"""How well the judge agrees with human labels: accuracy and Cohen's kappa.
+
+Accuracy alone flatters a judge on lopsided labels (if 90% of outputs are good, a
+judge that always says yes agrees 90% of the time). Kappa corrects for agreement
+expected by chance: 1 is perfect, 0 is no better than chance.
+"""
+
+from collections.abc import Sequence
+
+from pydantic import BaseModel, ConfigDict
+
+
+class Agreement(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    n: int
+    n_agree: int
+    accuracy: float | None
+    kappa: float | None  # None when chance agreement is total (one answer only)
+
+
+def agreement(pairs: Sequence[tuple[bool, bool]]) -> Agreement:
+    """Agreement over (human, judge) verdict pairs."""
+    n = len(pairs)
+    if n == 0:
+        return Agreement(n=0, n_agree=0, accuracy=None, kappa=None)
+    n_agree = sum(1 for human, judge in pairs if human == judge)
+    observed = n_agree / n
+    human_yes = sum(1 for human, _ in pairs if human) / n
+    judge_yes = sum(1 for _, judge in pairs if judge) / n
+    chance = human_yes * judge_yes + (1 - human_yes) * (1 - judge_yes)
+    kappa = None if chance >= 1 else (observed - chance) / (1 - chance)
+    return Agreement(n=n, n_agree=n_agree, accuracy=observed, kappa=kappa)

hemp/calibrate/labels.py

@@ -0,0 +1,143 @@
+"""Human labels for judge questions, and choosing which outputs to label.
+
+A label is keyed by exactly what the judge sees (question, task, output, reference),
+so it stays valid across runs. Rewording a question means its old labels no longer
+apply, which is intended: the criterion changed.
+"""
+
+import random
+from collections import defaultdict
+from collections.abc import Sequence
+from pathlib import Path
+
+import dspy
+from pydantic import BaseModel, ConfigDict
+
+from hemp.dataset import Row
+from hemp.hashing import canonical_json, sha256_text
+from hemp.images import image_columns
+from hemp.metrics.judge import to_dspy_image
+from hemp.plan.schema import EvalPlan
+from hemp.prompt import template_fields
+from hemp.runner.judging import reference_for
+from hemp.runner.results import RunResult
+from hemp.runner.target import build_prompt, prompt_images, prompt_text
+
+JUDGE_INPUTS = ("task", "output", "reference", "question", "images")
+
+
+class _Model(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+def label_key(
+    question: str, task: str, output: str, reference: str, images: Sequence[str] = ()
+) -> str:
+    parts: list[object] = [question, task, output, reference]
+    if images:
+        parts.append(list(images))
+    return sha256_text(canonical_json(parts))
+
+
+class LabelItem(_Model):
+    """One judge question about one stored output, waiting for a human answer."""
+
+    question_id: str
+    question: str
+    task: str
+    output: str
+    reference: str
+    images: tuple[str, ...] = ()  # image files or URLs in the task, in order
+    judge_passed: bool  # what the judge said in the run (hidden while labeling)
+    run_id: int | None
+    variant: str
+    example: int
+
+    @property
+    def key(self) -> str:
+        return label_key(self.question, self.task, self.output, self.reference, self.images)
+
+
+class Label(_Model):
+    question: str
+    task: str
+    output: str
+    reference: str
+    passed: bool  # the human's answer
+    images: tuple[str, ...] = ()
+
+    @property
+    def key(self) -> str:
+        return label_key(self.question, self.task, self.output, self.reference, self.images)
+
+    def as_example(self) -> dspy.Example:
+        return dspy.Example(
+            task=self.task,
+            output=self.output,
+            reference=self.reference,
+            question=self.question,
+            images=[to_dspy_image(i) for i in self.images],
+            answer=self.passed,
+        ).with_inputs(*JUDGE_INPUTS)
+
+
+def label_candidates(
+    run: RunResult,
+    rows: Sequence[Row],
+    prompt: str,
+    plan: EvalPlan,
+    *,
+    skip: set[str] | None = None,
+    n: int = 20,
+    seed: int = 0,
+) -> list[LabelItem]:
+    """Up to `n` outputs to label, spread across questions and across the judge's yes/no.
+
+    Outputs from baselines are included: they're often bad, which gives the judge
+    examples of failures to learn from. Items whose key is in `skip` are left out.
+    """
+    skip = skip or set()
+    images = image_columns(rows, {*template_fields(prompt), *plan.input_columns})
+    base_dir = Path(run.info.data_dir) if run.info.data_dir else None
+    groups: dict[tuple[str, bool], list[LabelItem]] = defaultdict(list)
+    seen: set[str] = set()
+    for criterion in plan.criteria:
+        if run.questions.get(criterion.id) != criterion.question:
+            continue  # this run judged a different version of the question
+        for variant in run.variants:
+            for ex in variant.examples:
+                outcome = next((o for o in ex.checks if o.check_id == criterion.id), None)
+                if outcome is None or outcome.passed is None or ex.error is not None:
+                    continue
+                row = rows[ex.index]
+                reference = reference_for(criterion, row)
+                if reference is None:
+                    continue
+                content = build_prompt(prompt, row, images, base_dir)
+                item = LabelItem(
+                    question_id=criterion.id,
+                    question=criterion.question,
+                    task=prompt_text(content),
+                    output=ex.output,
+                    reference=reference,
+                    images=tuple(i.location for i in prompt_images(content)),
+                    judge_passed=outcome.passed,
+                    run_id=run.id,
+                    variant=variant.name,
+                    example=ex.index,
+                )
+                if item.key in skip or item.key in seen:
+                    continue
+                seen.add(item.key)
+                groups[(criterion.id, outcome.passed)].append(item)
+
+    rng = random.Random(seed)
+    queues = [groups[k] for k in sorted(groups)]
+    for queue in queues:
+        rng.shuffle(queue)
+    picked: list[LabelItem] = []
+    while len(picked) < n and any(queues):
+        for queue in queues:
+            if queue and len(picked) < n:
+                picked.append(queue.pop())
+    return picked

hemp/calibrate/optimize.py

@@ -0,0 +1,213 @@
+"""Measure the judge against human labels and tune it with DSPy.
+
+With enough labels, they're split in two: the judge is tuned on one half (DSPy
+picks worked examples to show it) and both the default and the tuned judge are
+tested on the other half, which tuning never saw. The tuned judge is only kept if
+it agrees with the human more on that held-out half.
+"""
+
+import contextlib
+import io
+import random
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from typing import Any
+
+import dspy
+from pydantic import BaseModel, ConfigDict
+
+from hemp.calibrate.agreement import Agreement, agreement
+from hemp.calibrate.labels import Label
+from hemp.metrics.judge import (
+    CalibratedJudge,
+    Judge,
+    _as_bool,
+    make_judge,
+    module_state,
+    to_dspy_image,
+)
+from hemp.runner.parallel import run_all
+
+MIN_LABELS_TO_TUNE = 8
+MAX_EXAMPLES_SHOWN = 4  # worked examples the tuned judge sees in its prompt
+
+
+class CalibrationFailed(RuntimeError):
+    """The judge couldn't be measured, e.g. its model is unreachable."""
+
+
+class QuestionAgreement(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    question: str
+    n_labels: int
+    before: Agreement  # the default judge
+    after: Agreement | None = None  # the tuned judge, on the same held-out labels
+
+
+class CalibrationReport(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    n_labels: int
+    n_train: int
+    n_test: int  # labels the agreement numbers are measured on
+    before: Agreement
+    after: Agreement | None
+    by_question: list[QuestionAgreement]
+    n_failed: int  # judge calls that failed and were left out
+    tuned: CalibratedJudge | None  # set only when tuning improved agreement
+
+    @property
+    def improved(self) -> bool:
+        return self.tuned is not None
+
+
+def calibrate(
+    labels: Sequence[Label],
+    judge_lm: Any,
+    *,
+    judge_model: str,
+    workers: int = 4,
+    seed: int = 0,
+    now: datetime | None = None,
+) -> CalibrationReport:
+    """Measure the default judge on the labels and, with enough of them, tune it."""
+    if not labels:
+        raise CalibrationFailed("There are no labels yet. Label some outputs first.")
+    base = make_judge()
+    base.set_lm(judge_lm)
+
+    if len(labels) < MIN_LABELS_TO_TUNE:
+        verdicts = _predict(base, labels, workers)
+        return CalibrationReport(
+            n_labels=len(labels),
+            n_train=0,
+            n_test=len(labels),
+            before=_agree(labels, verdicts),
+            after=None,
+            by_question=_by_question(labels, verdicts, None),
+            n_failed=sum(1 for v in verdicts if v is None),
+            tuned=None,
+        )
+
+    train, test = split(labels, seed)
+    before_verdicts = _predict(base, test, workers)
+    tuned_judge = _tune(train, judge_lm, seed)
+    after_verdicts = _predict(tuned_judge, test, workers)
+    before, after = _agree(test, before_verdicts), _agree(test, after_verdicts)
+    improved = (
+        after.accuracy is not None
+        and before.accuracy is not None
+        and after.accuracy > before.accuracy
+    )
+    tuned = (
+        CalibratedJudge(
+            judge_model=judge_model,
+            state=module_state(tuned_judge),
+            created_at=now or datetime.now(UTC).replace(microsecond=0),
+            n_labels=len(labels),
+            n_test=len(test),
+            agreement=after.accuracy,
+            kappa=after.kappa,
+        )
+        if improved
+        else None
+    )
+    return CalibrationReport(
+        n_labels=len(labels),
+        n_train=len(train),
+        n_test=len(test),
+        before=before,
+        after=after,
+        by_question=_by_question(test, before_verdicts, after_verdicts),
+        n_failed=sum(1 for v in [*before_verdicts, *after_verdicts] if v is None),
+        tuned=tuned,
+    )
+
+
+def split(labels: Sequence[Label], seed: int = 0) -> tuple[list[Label], list[Label]]:
+    """Half for tuning, half for testing, keeping yes and no labels in both halves."""
+    rng = random.Random(seed)
+    train: list[Label] = []
+    test: list[Label] = []
+    for answer in (True, False):
+        group = sorted((lb for lb in labels if lb.passed is answer), key=lambda lb: lb.key)
+        rng.shuffle(group)
+        half = len(group) // 2
+        train += group[half:]
+        test += group[:half]
+    return train, test
+
+
+def _tune(train: Sequence[Label], judge_lm: Any, seed: int) -> Judge:
+    def metric(example: dspy.Example, pred: dspy.Prediction, trace: object = None) -> bool:
+        try:
+            return _as_bool(pred.answer) is bool(example.answer)
+        except ValueError:
+            return False
+
+    student = make_judge()
+    student.set_lm(judge_lm)
+    optimizer = dspy.BootstrapFewShot(
+        metric=metric,
+        max_bootstrapped_demos=MAX_EXAMPLES_SHOWN,
+        max_labeled_demos=MAX_EXAMPLES_SHOWN,
+    )
+    trainset = [lb.as_example() for lb in train]
+    random.Random(seed).shuffle(trainset)
+    # redirect_stdout: BootstrapFewShot prints a progress line we don't want in HEMP's output.
+    with dspy.context(lm=judge_lm), contextlib.redirect_stdout(io.StringIO()):
+        tuned: Judge = optimizer.compile(student, trainset=trainset)
+    tuned.set_lm(judge_lm)
+    return tuned
+
+
+def _predict(judge: Judge, labels: Sequence[Label], workers: int) -> list[bool | None]:
+    def one(label: Label) -> bool | None:
+        try:
+            pred = judge(
+                task=label.task,
+                output=label.output,
+                question=label.question,
+                reference=label.reference,
+                images=[to_dspy_image(i) for i in label.images],
+            )
+            return _as_bool(pred.answer)
+        except Exception:
+            return None
+
+    def check_first(verdict: bool | None) -> None:
+        if verdict is None:
+            raise CalibrationFailed(
+                "The judge couldn't answer. Is its model running (`ollama serve`)? "
+                "You can pick another one with --judge-model."
+            )
+
+    return run_all(one, labels, workers=workers, check_first=check_first)
+
+
+def _agree(labels: Sequence[Label], verdicts: Sequence[bool | None]) -> Agreement:
+    return agreement(
+        [(lb.passed, v) for lb, v in zip(labels, verdicts, strict=True) if v is not None]
+    )
+
+
+def _by_question(
+    labels: Sequence[Label],
+    before: Sequence[bool | None],
+    after: Sequence[bool | None] | None,
+) -> list[QuestionAgreement]:
+    questions = list(dict.fromkeys(lb.question for lb in labels))
+    out = []
+    for question in questions:
+        idx = [i for i, lb in enumerate(labels) if lb.question == question]
+        subset = [labels[i] for i in idx]
+        out.append(
+            QuestionAgreement(
+                question=question,
+                n_labels=len(idx),
+                before=_agree(subset, [before[i] for i in idx]),
+                after=_agree(subset, [after[i] for i in idx]) if after is not None else None,
+            )
+        )
+    return out

hemp/calibrate/render.py

@@ -0,0 +1,87 @@
+"""Terminal output for `hemp calibrate`: labeling screens and the agreement report."""
+
+from rich.console import Group, RenderableType
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from hemp.calibrate.agreement import Agreement
+from hemp.calibrate.labels import LabelItem
+from hemp.calibrate.optimize import MIN_LABELS_TO_TUNE, CalibrationReport
+from hemp.metrics.judge import NO_REFERENCE
+
+TASK_PREVIEW_CHARS = 1500
+
+
+def render_item(item: LabelItem, number: int, total: int) -> RenderableType:
+    task = item.task
+    if len(task) > TASK_PREVIEW_CHARS:
+        task = task[:TASK_PREVIEW_CHARS] + "\n[...]"
+    source = "your prompt" if item.variant == "prompt" else f"baseline {item.variant}"
+    parts: list[RenderableType] = [
+        Text.assemble(("Question: ", "bold"), (item.question, "bold cyan")),
+        Text(""),
+        Text("The task the model got:", style="bold"),
+        Text(task, style="dim"),
+    ]
+    if item.images:
+        parts += [Text(""), Text("Images in the task (open them to look):", style="bold")]
+        parts += [Text(f"  [image {n}] {path}") for n, path in enumerate(item.images, start=1)]
+    if item.reference != NO_REFERENCE:
+        parts += [Text(""), Text.assemble(("Reference answer: ", "bold"), item.reference)]
+    parts += [Text(""), Text(f"Output ({source}):", style="bold"), Text(item.output)]
+    return Panel(Group(*parts), title=f"Label {number} of {total}", title_align="left")
+
+
+def _cell(a: Agreement | None) -> Text:
+    if a is None:
+        return Text("-", style="dim")
+    if a.accuracy is None:
+        return Text("no verdicts", style="dim")
+    kappa = "n/a" if a.kappa is None else f"{a.kappa:.2f}"
+    return Text.assemble(f"{a.accuracy:.0%} ({a.n_agree}/{a.n})", (f"  kappa {kappa}", "dim"))
+
+
+def render_report(report: CalibrationReport) -> RenderableType:
+    tuned = report.after is not None
+    table = Table(header_style="bold", title_justify="left")
+    table.add_column("Question", overflow="fold")
+    table.add_column("Labels", justify="right")
+    table.add_column("Default judge" if tuned else "Judge agrees with you", justify="right")
+    if tuned:
+        table.add_column("Tuned judge", justify="right")
+    for q in report.by_question:
+        cells = [_cell(q.before)] + ([_cell(q.after)] if tuned else [])
+        table.add_row(q.question, str(q.n_labels), *cells)
+    table.add_section()
+    overall = [_cell(report.before)] + ([_cell(report.after)] if tuned else [])
+    table.add_row(Text("All questions", style="bold"), str(report.n_test), *overall)
+
+    if tuned:
+        intro = (
+            f"Tuned the judge on {report.n_train} of your {report.n_labels} labels and tested "
+            f"both judges on the other {report.n_test}, which tuning never saw."
+        )
+    else:
+        intro = f"How often the judge agrees with your {report.n_labels} label(s):"
+    parts: list[RenderableType] = [Text(intro), table]
+    parts.append(
+        Text(
+            "Kappa corrects for agreement by chance: 1 is perfect, 0 is no better than "
+            "guessing, above 0.6 is good.",
+            style="dim",
+        )
+    )
+    if report.n_failed:
+        parts.append(
+            Text(f"{report.n_failed} judge call(s) failed and were left out.", style="yellow")
+        )
+    if not tuned:
+        parts.append(
+            Text(
+                f"Label at least {MIN_LABELS_TO_TUNE} outputs to tune the judge "
+                f"(you have {report.n_labels}).",
+                style="yellow",
+            )
+        )
+    return Group(*parts)