evalcore 0.1.0__py3-none-any.whl

evalkit/runner.py

@@ -0,0 +1,283 @@
+"""The runner: execute one suite x one variant over a dataset -> RunResult.
+
+Generic and consumer-agnostic. For each case it invokes the adapter
+``n_samples`` times, runs the per-case graders, then runs the aggregate graders
+over the whole result set, and returns the aggregated ``Scorecard`` together
+with every per-sample ``CaseResult`` (outputs, artifacts, scores) so nothing
+downstream has to re-run the suite to inspect individual generations.
+
+``suite.concurrency > 1`` runs the (case, sample) invocations concurrently
+under a semaphore; the adapter and per-case graders must then tolerate
+concurrent calls.
+"""
+
+import asyncio
+import inspect
+import statistics
+import uuid
+
+from evalkit import loader, models, store
+from evalkit import retry as retry_mod
+from evalkit.adapters import base as adapters_base
+from evalkit.graders import base as graders_base
+
+
+async def _invoke_with_retry(
+    adapter,
+    case: models.Case,
+    variant: models.Variant,
+    retry: retry_mod.RetryConfig,
+) -> models.Output:
+    """Invoke the adapter, retrying transient failures with backoff.
+
+    Retries only an ``Output`` the adapter flagged ``retryable`` (a 429, 5xx,
+    or network error); a terminal error or a success returns immediately. The
+    adapter contract still holds - a failed call sets ``Output.error`` rather
+    than raising - so this loop reads the flag, it does not catch exceptions.
+    Returns the last ``Output`` (errored) once the attempt budget is spent.
+    """
+    attempt = 1
+    while True:
+        output = await adapter.invoke(case, variant)
+        if not (output.error and output.retryable):
+            return output
+        if attempt >= retry.max_attempts:
+            return output
+        await asyncio.sleep(retry_mod.backoff_delay(attempt, retry))
+        attempt += 1
+
+
+def _aggregate_metrics(
+    results: list[models.CaseResult], agg_scores: list[models.Score]
+) -> dict[str, models.MetricValue]:
+    """Fold per-case scores (mean) and aggregate scores (as-is) to metrics."""
+    metrics: dict[str, models.MetricValue] = {}
+
+    by_metric: dict[str, list[float]] = {}
+    for result in results:
+        for score in result.scores:
+            if score.value is not None:
+                by_metric.setdefault(score.metric, []).append(score.value)
+    for metric, values in by_metric.items():
+        metrics[metric] = models.MetricValue(
+            metric=metric,
+            value=statistics.fmean(values) if values else None,
+            stdev=statistics.stdev(values) if len(values) > 1 else None,
+            kind='mean',
+            n=len(values),
+        )
+
+    for score in agg_scores:
+        metrics[score.metric] = models.MetricValue(
+            metric=score.metric,
+            value=score.value,
+            kind='aggregate',
+            n=len(results),
+        )
+    return metrics
+
+
+async def run_suite(
+    suite: loader.SuiteConfig,
+    variant_name: str,
+    *,
+    mode: str | None = None,
+    grader_mode: str | None = None,
+    revision: str | None = None,
+    created_at: str | None = None,
+    checkpoint: str | None = None,
+    resume: bool = False,
+) -> models.RunResult:
+    """Run ``variant_name`` of ``suite``; return scorecard + all results.
+
+    ``mode`` overrides ``suite.mode_default`` and drives the ADAPTER
+    (``'replay'`` swaps the configured adapter for the replay adapter backed
+    by ``replay_fixtures``). ``grader_mode`` drives the GRADERS independently
+    (defaults to ``mode``); set it to run live judges over recorded outputs
+    (``mode='replay', grader_mode='http'``) or replayed judges over a live
+    target, without coupling the two.
+
+    ``checkpoint`` is a JSONL file the runner appends each completed
+    ``(case, sample)`` to; if ``resume`` and it already holds results for the
+    *same* suite/dataset/variant (verified by content hash), those are loaded
+    and their invocations skipped, so an interrupted run continues instead of
+    restarting. Without ``resume`` an existing checkpoint is overwritten.
+    """
+    if variant_name not in suite.variants:
+        raise KeyError(
+            f'variant {variant_name!r} not in suite '
+            f'(have {sorted(suite.variants)})'
+        )
+    mode = mode or suite.mode_default
+    grader_mode = grader_mode or mode
+    variant = models.Variant(
+        name=variant_name, knobs=suite.variants[variant_name]
+    )
+
+    if mode == 'replay':
+        if not suite.replay_fixtures:
+            raise ValueError('replay mode requires suite.replay_fixtures')
+        adapter = adapters_base.build_adapter(
+            {'type': 'replay', 'fixtures': suite.replay_fixtures}
+        )
+    else:
+        adapter = adapters_base.build_adapter(suite.adapter)
+
+    per_case_graders, aggregate_graders = graders_base.build_graders(
+        suite.graders
+    )
+    # Let graders that care (e.g. the LLM judge) bind to grader_mode so they
+    # pick a live vs. replay client - independently of the adapter's mode -
+    # and to the suite's retry policy so a transient judge-client error backs
+    # off like an adapter call. Graders without the hook are left untouched.
+    for grader in (*per_case_graders, *aggregate_graders):
+        setter = getattr(grader, 'set_mode', None)
+        if callable(setter):
+            setter(grader_mode)
+        retry_setter = getattr(grader, 'set_retry', None)
+        if callable(retry_setter):
+            retry_setter(suite.retry)
+
+    cases = loader.load_cases(suite.dataset)
+    dataset_hash = loader.dataset_hash(cases)
+    run_id = uuid.uuid4().hex
+
+    # Resume: reuse already-completed (case, sample) results from a checkpoint
+    # of the *same* eval; otherwise start (or restart) the checkpoint fresh.
+    done: dict[tuple[str, int], models.CaseResult] = {}
+    if checkpoint:
+        meta = store.checkpoint_meta(checkpoint) if resume else None
+        if meta:
+            mismatch = (
+                meta.get('suite_hash') != suite.suite_hash
+                or meta.get('dataset_hash') != dataset_hash
+                or meta.get('variant') != variant_name
+            )
+            if mismatch:
+                raise ValueError(
+                    'checkpoint is for a different suite/dataset/variant; '
+                    'cannot resume (delete it to start over)'
+                )
+            run_id = meta['run_id']
+            for result in store.read_checkpoint_results(checkpoint):
+                done[result.case.id, result.sample_idx] = result
+        else:
+            store.init_checkpoint(
+                checkpoint,
+                {
+                    'run_id': run_id,
+                    'suite_hash': suite.suite_hash,
+                    'dataset_hash': dataset_hash,
+                    'project': suite.project,
+                    'suite': suite.suite,
+                    'variant': variant_name,
+                },
+            )
+
+    async def _run_one(
+        case: models.Case, sample_idx: int
+    ) -> models.CaseResult:
+        output = await _invoke_with_retry(adapter, case, variant, suite.retry)
+        scores: list[models.Score] = []
+        for grader in per_case_graders:
+            graded = grader.grade(case, output)
+            if inspect.isawaitable(graded):
+                graded = await graded
+            scores.extend(graded)
+        result = models.CaseResult(
+            case=case,
+            variant_name=variant_name,
+            sample_idx=sample_idx,
+            output=output,
+            scores=scores,
+        )
+        # Record on completion so an interrupt leaves a resumable trail. The
+        # append is synchronous within this coroutine, so concurrent runs
+        # never interleave a half-written line.
+        if checkpoint:
+            store.append_checkpoint_result(checkpoint, result)
+        return result
+
+    jobs = [
+        (case, idx)
+        for case in cases
+        for idx in range(suite.n_samples)
+        if (case.id, idx) not in done
+    ]
+    try:
+        if suite.concurrency > 1:
+            semaphore = asyncio.Semaphore(suite.concurrency)
+
+            async def _bounded(
+                case: models.Case, idx: int
+            ) -> models.CaseResult:
+                async with semaphore:
+                    return await _run_one(case, idx)
+
+            fresh = list(
+                await asyncio.gather(
+                    *(_bounded(case, idx) for case, idx in jobs)
+                )
+            )
+        else:
+            fresh = [await _run_one(case, idx) for case, idx in jobs]
+    finally:
+        # Adapters holding resources (browser, injected session, pooled
+        # connections) may expose an optional async ``aclose`` hook.
+        closer = getattr(adapter, 'aclose', None)
+        if callable(closer):
+            closed = closer()
+            if inspect.isawaitable(closed):
+                await closed
+
+    # Merge resumed + fresh results into dataset order (case position, then
+    # sample), so the run is identical regardless of completion order or how
+    # many resumes it took.
+    position = {case.id: i for i, case in enumerate(cases)}
+    results = sorted(
+        [*done.values(), *fresh],
+        key=lambda r: (position.get(r.case.id, len(cases)), r.sample_idx),
+    )
+
+    agg_scores: list[models.Score] = []
+    for grader in aggregate_graders:
+        agg_scores.extend(grader.aggregate(results))
+
+    # Provenance: a grader may expose a ``judge_version`` pin (the LLM judge
+    # does). Collect it onto the scorecard so a judge change re-baselines;
+    # multiple judge graders join with ';'. Generic - non-judge graders have
+    # no such attribute and contribute nothing.
+    judge_pins = [
+        pin
+        for grader in (*per_case_graders, *aggregate_graders)
+        if (pin := getattr(grader, 'judge_version', None))
+    ]
+
+    scorecard = models.Scorecard(
+        run_id=run_id,
+        project=suite.project,
+        suite=suite.suite,
+        variant=variant,
+        dataset_version=suite.dataset_version,
+        model_id=variant.knobs.get('model'),
+        prompt_version=variant.knobs.get('prompt_version'),
+        judge_version=';'.join(judge_pins) or None,
+        revision=revision,
+        suite_hash=suite.suite_hash,
+        dataset_hash=dataset_hash,
+        mode=mode,
+        n_samples=suite.n_samples,
+        n_cases=len(cases),
+        created_at=created_at,
+        metrics=_aggregate_metrics(results, agg_scores),
+    )
+    return models.RunResult(
+        run_id=run_id, scorecard=scorecard, results=results
+    )
+
+
+def run_suite_sync(
+    suite: loader.SuiteConfig, variant_name: str, **kwargs
+) -> models.RunResult:
+    """Blocking convenience wrapper around :func:`run_suite`."""
+    return asyncio.run(run_suite(suite, variant_name, **kwargs))

evalkit/store.py

@@ -0,0 +1,286 @@
+"""Scorecard/comparison persistence and the results-store export seam.
+
+Scorecards and comparisons serialize to JSON (CI artifacts, local files). For
+trend tracking you can store them in a column store such as ClickHouse, keyed
+by ``project``/``suite``; rather than couple the engine to any particular
+driver, this module flattens a scorecard into a stable row shape and writes it
+to a JSONL **outbox** a separate shipper drains. Swap ``JsonlOutboxExporter``
+for a real database client without touching the runner or any consumer.
+
+The emitted rows follow a no-``Nullable`` convention that maps cleanly onto a
+column store: a missing metric value is the sentinel pair
+``(value=0, has_value=false)`` and ``passed`` is the tri-state string
+``'true'|'false'|'null'``, so a JSONEachRow-style feed maps straight onto a
+flat ``eval_runs``/``eval_scores`` schema.
+"""
+
+import json
+import pathlib
+
+from evalkit import models
+
+
+def write_scorecard(
+    path: str | pathlib.Path, scorecard: models.Scorecard
+) -> None:
+    """Write a scorecard as pretty JSON."""
+    pathlib.Path(path).write_text(
+        scorecard.model_dump_json(indent=2), encoding='utf-8'
+    )
+
+
+def read_scorecard(path: str | pathlib.Path) -> models.Scorecard:
+    """Read a scorecard previously written by :func:`write_scorecard`."""
+    return models.Scorecard.model_validate_json(
+        pathlib.Path(path).read_text(encoding='utf-8')
+    )
+
+
+def load_scorecard(path: str | pathlib.Path) -> models.Scorecard:
+    """Read a scorecard from either a scorecard file or a full-run file.
+
+    Accepts both ``run --out`` (a bare Scorecard) and ``run --run-out`` (a
+    RunResult wrapping one), so a comparison can consume whichever artifact
+    a prior run left behind.
+    """
+    data = json.loads(pathlib.Path(path).read_text(encoding='utf-8'))
+    if isinstance(data, dict) and 'scorecard' in data and 'results' in data:
+        return models.Scorecard.model_validate(data['scorecard'])
+    return models.Scorecard.model_validate(data)
+
+
+def write_run(path: str | pathlib.Path, run: models.RunResult) -> None:
+    """Write a full run (scorecard + every per-sample result) as JSON.
+
+    This is the persisted ground truth behind a scorecard: transcript
+    review, human rating, and judge-agreement analysis all read it back
+    rather than re-running the suite.
+    """
+    pathlib.Path(path).write_text(
+        run.model_dump_json(indent=2), encoding='utf-8'
+    )
+
+
+def read_run(path: str | pathlib.Path) -> models.RunResult:
+    """Read a run previously written by :func:`write_run`."""
+    return models.RunResult.model_validate_json(
+        pathlib.Path(path).read_text(encoding='utf-8')
+    )
+
+
+# -- run checkpointing (idempotent resume) ----------------------------------
+#
+# A checkpoint is a JSONL file: line 1 is a meta header (run_id + the content
+# hashes that identify *which* eval it belongs to), and each subsequent line is
+# one completed ``CaseResult``. The runner appends a line as each (case,
+# sample) finishes, so an interrupted run leaves a valid partial file that a
+# ``--resume`` re-run reads back to skip work already done.
+
+
+def init_checkpoint(path: str | pathlib.Path, meta: dict) -> None:
+    """Start a fresh checkpoint file with its meta header line (truncating)."""
+    path = pathlib.Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', encoding='utf-8') as handle:
+        handle.write(json.dumps(meta) + '\n')
+
+
+def checkpoint_meta(path: str | pathlib.Path) -> dict | None:
+    """The checkpoint meta header, or ``None`` if the file is absent/empty."""
+    path = pathlib.Path(path)
+    if not path.is_file():
+        return None
+    with path.open(encoding='utf-8') as handle:
+        first = handle.readline()
+    return json.loads(first) if first.strip() else None
+
+
+def append_checkpoint_result(
+    path: str | pathlib.Path, result: models.CaseResult
+) -> None:
+    """Append one completed per-sample result to the checkpoint file."""
+    with pathlib.Path(path).open('a', encoding='utf-8') as handle:
+        handle.write(result.model_dump_json() + '\n')
+
+
+def read_checkpoint_results(
+    path: str | pathlib.Path,
+) -> list[models.CaseResult]:
+    """The completed results recorded in a checkpoint (after the meta line)."""
+    path = pathlib.Path(path)
+    if not path.is_file():
+        return []
+    lines = path.read_text(encoding='utf-8').splitlines()
+    return [
+        models.CaseResult.model_validate_json(line)
+        for line in lines[1:]
+        if line.strip()
+    ]
+
+
+def write_comparison(
+    path: str | pathlib.Path, comparison: models.Comparison
+) -> None:
+    """Write a comparison as pretty JSON."""
+    pathlib.Path(path).write_text(
+        comparison.model_dump_json(indent=2), encoding='utf-8'
+    )
+
+
+def append_rating(path: str | pathlib.Path, rating: models.Rating) -> None:
+    """Append one human rating to a JSONL ratings file (creating it)."""
+    path = pathlib.Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('a', encoding='utf-8') as handle:
+        handle.write(rating.model_dump_json() + '\n')
+
+
+def read_ratings(path: str | pathlib.Path) -> list[models.Rating]:
+    """Read a JSONL ratings file (the open human-rating interchange format).
+
+    Rows may come from the ``rate`` web app or any external tool that emits
+    the same shape - one JSON object per line.
+    """
+    path = pathlib.Path(path)
+    if not path.is_file():
+        return []
+    return [
+        models.Rating.model_validate_json(line)
+        for line in path.read_text(encoding='utf-8').splitlines()
+        if line.strip()
+    ]
+
+
+def append_preference(
+    path: str | pathlib.Path, preference: models.Preference
+) -> None:
+    """Append one side-by-side preference to a JSONL file (creating it)."""
+    path = pathlib.Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('a', encoding='utf-8') as handle:
+        handle.write(preference.model_dump_json() + '\n')
+
+
+def read_preferences(path: str | pathlib.Path) -> list[models.Preference]:
+    """Read a JSONL preferences file (the open A-vs-B interchange format).
+
+    Rows may come from the ``rank`` web app or any external tool that emits
+    the same shape - one JSON object per line.
+    """
+    path = pathlib.Path(path)
+    if not path.is_file():
+        return []
+    return [
+        models.Preference.model_validate_json(line)
+        for line in path.read_text(encoding='utf-8').splitlines()
+        if line.strip()
+    ]
+
+
+def scorecard_rows(scorecard: models.Scorecard) -> list[dict]:
+    """Flatten a scorecard into one row per metric (the store schema).
+
+    Matches the ``eval_runs``/``eval_scores`` columns in the platform design:
+    every row carries the full reproducibility key so rows are self-describing
+    in a multi-tenant table.
+    """
+    key = _run_key(scorecard)
+    return [
+        {
+            **key,
+            'metric': metric.metric,
+            # No-Nullable store convention: a missing value is the sentinel
+            # pair (0, has_value=false), not null (docs/clickhouse-schema.sql).
+            'value': metric.value if metric.value is not None else 0.0,
+            'has_value': metric.value is not None,
+            'stdev': metric.stdev if metric.stdev is not None else 0.0,
+            'has_stdev': metric.stdev is not None,
+            'metric_kind': metric.kind,
+            'n': metric.n,
+        }
+        for metric in scorecard.metrics.values()
+    ]
+
+
+def _run_key(scorecard: models.Scorecard) -> dict:
+    """The reproducibility key carried on every store row."""
+    return {
+        'run_id': scorecard.run_id,
+        'project': scorecard.project,
+        'suite': scorecard.suite,
+        'variant': scorecard.variant.name,
+        'dataset_version': scorecard.dataset_version,
+        'model_id': scorecard.model_id,
+        'prompt_version': scorecard.prompt_version,
+        'judge_version': scorecard.judge_version,
+        'revision': scorecard.revision,
+        'suite_hash': scorecard.suite_hash,
+        'dataset_hash': scorecard.dataset_hash,
+        'mode': scorecard.mode,
+        'created_at': scorecard.created_at,
+    }
+
+
+def score_rows(run: models.RunResult) -> list[dict]:
+    """Flatten a run into one row per per-case score (``eval_scores``).
+
+    The per-sample companion to :func:`scorecard_rows`: every individual
+    grader score, keyed by ``run_id``/``case_id``/``sample_idx``, so the
+    store keeps the raw observations the scorecard means were computed
+    from (needed for variance, drill-down, and judge-vs-human analysis).
+    """
+    key = _run_key(run.scorecard)
+    rows: list[dict] = []
+    for result in run.results:
+        for score in result.scores:
+            rows.append(
+                {
+                    **key,
+                    'case_id': result.case.id,
+                    'sample_idx': result.sample_idx,
+                    'grader': score.grader,
+                    'metric': score.metric,
+                    'value': score.value if score.value is not None else 0.0,
+                    'has_value': score.value is not None,
+                    # Tri-state (Enum8 'true'/'false'/'null') per the store
+                    # convention: deterministic graders set passed, judges
+                    # leave it null.
+                    'passed': (
+                        'true'
+                        if score.passed is True
+                        else 'false'
+                        if score.passed is False
+                        else 'null'
+                    ),
+                    'detail': score.detail or '',
+                    'errored': result.output.error is not None,
+                }
+            )
+    return rows
+
+
+class JsonlOutboxExporter:
+    """Append scorecard rows to a JSONL outbox for a ClickHouse shipper.
+
+    A no-network stand-in for direct ClickHouse ingestion: real deployments
+    point a shipper at this file (or replace this class with a ClickHouse
+    client implementing the same ``export`` method).
+    """
+
+    def __init__(self, outbox_path: str | pathlib.Path):
+        self.outbox_path = pathlib.Path(outbox_path)
+
+    def _append(self, rows: list[dict]) -> int:
+        self.outbox_path.parent.mkdir(parents=True, exist_ok=True)
+        with self.outbox_path.open('a', encoding='utf-8') as handle:
+            for row in rows:
+                handle.write(json.dumps(row) + '\n')
+        return len(rows)
+
+    def export(self, scorecard: models.Scorecard) -> int:
+        """Append one row per metric; return the number of rows written."""
+        return self._append(scorecard_rows(scorecard))
+
+    def export_scores(self, run: models.RunResult) -> int:
+        """Append one row per per-case score (the ``eval_scores`` feed)."""
+        return self._append(score_rows(run))

evalkit/sweep.py

@@ -0,0 +1,93 @@
+"""N-way sweeps: run many variants of a suite and rank them.
+
+Where :mod:`compare` answers "candidate vs baseline", a sweep answers
+"which of these N variants is best" - a model x prompt-version matrix.
+It reuses the per-variant runner unchanged: run every named variant, then
+tabulate one leaderboard (metric x variant) and a ranking by the suite's
+win metric. Purely a run-orchestration + summary layer; no new seams.
+"""
+
+from evalkit import loader, models, runner
+
+
+async def run_sweep(
+    suite: loader.SuiteConfig,
+    variant_names: list[str] | None = None,
+    *,
+    mode: str | None = None,
+    grader_mode: str | None = None,
+    revision: str | None = None,
+    created_at: str | None = None,
+) -> list[models.RunResult]:
+    """Run each named variant (default: all) and return their runs."""
+    names = variant_names or list(suite.variants)
+    runs: list[models.RunResult] = []
+    for name in names:
+        runs.append(
+            await runner.run_suite(
+                suite,
+                name,
+                mode=mode,
+                grader_mode=grader_mode,
+                revision=revision,
+                created_at=created_at,
+            )
+        )
+    return runs
+
+
+def summarize_sweep(
+    runs: list[models.RunResult], thresholds: dict | None = None
+) -> models.SweepResult:
+    """Fold sweep runs into a ranked leaderboard by the win metric."""
+    thresholds = thresholds or {}
+    win_metric = thresholds.get('win_metric')
+    higher = thresholds.get('win_higher_is_better', True)
+    scorecards = [run.scorecard for run in runs]
+
+    all_metrics = sorted(
+        {metric for card in scorecards for metric in card.metrics}
+    )
+    matrix: dict[str, dict[str, float | None]] = {}
+    for metric in all_metrics:
+        matrix[metric] = {}
+        for card in scorecards:
+            found = card.metrics.get(metric)
+            matrix[metric][card.variant.name] = found.value if found else None
+
+    entries: list[models.SweepEntry] = []
+    if win_metric:
+        pairs = [
+            (
+                card.variant.name,
+                (
+                    card.metrics[win_metric].value
+                    if win_metric in card.metrics
+                    else None
+                ),
+            )
+            for card in scorecards
+        ]
+
+        def sort_key(pair: tuple[str, float | None]):
+            value = pair[1]
+            if value is None:
+                return (1, 0.0)  # missing metric sorts last
+            return (0, -value if higher else value)
+
+        for rank, (name, value) in enumerate(
+            sorted(pairs, key=sort_key), start=1
+        ):
+            entries.append(
+                models.SweepEntry(variant=name, win_value=value, rank=rank)
+            )
+
+    first = scorecards[0]
+    return models.SweepResult(
+        project=first.project,
+        suite=first.suite,
+        win_metric=win_metric,
+        win_higher_is_better=higher,
+        entries=entries,
+        matrix=matrix,
+    )