evalcore 0.1.0__py3-none-any.whl

evalkit/refs.py

@@ -0,0 +1,71 @@
+"""Reference resolution shared by adapters and graders.
+
+Cases, variants, and outputs are *opaque* to the engine - only adapters and
+graders crack them open, and they do so through string references like
+``$input.content`` (adapter request bodies) or ``output.verdict`` (grader
+field selectors). Keeping this resolution in one place is what lets the core
+stay ignorant of any consumer's data shape.
+"""
+
+import typing
+
+_MISSING = object()
+
+
+def resolve_path(obj: typing.Any, path: str) -> typing.Any:
+    """Traverse ``obj`` along a dotted ``path``.
+
+    Supports dict keys, list indices (numeric segments), and object
+    attributes, in that order of preference. Returns ``None`` when any
+    segment is missing rather than raising, so a grader/adapter selector
+    that points at an absent field degrades to ``None``.
+    """
+    current = obj
+    for segment in path.split('.'):
+        if current is None:
+            return None
+        if isinstance(current, dict):
+            current = current.get(segment, _MISSING)
+        elif (
+            isinstance(current, (list, tuple))
+            and segment.lstrip('-').isdigit()
+        ):
+            index = int(segment)
+            if -len(current) <= index < len(current):
+                current = current[index]
+            else:
+                current = _MISSING
+        else:
+            current = getattr(current, segment, _MISSING)
+        if current is _MISSING:
+            return None
+    return current
+
+
+def resolve_ref(context: dict, ref: str) -> typing.Any:
+    """Resolve a ``root.path`` reference against a context dict.
+
+    ``context`` maps root names (``input``, ``expected``, ``variant``,
+    ``output``, ``case``) to the corresponding objects.
+    """
+    root, _, rest = ref.partition('.')
+    if root not in context:
+        return None
+    base = context[root]
+    return resolve_path(base, rest) if rest else base
+
+
+def build_value(template: typing.Any, context: dict) -> typing.Any:
+    """Render a (possibly nested) template, substituting ``$ref`` strings.
+
+    A string beginning with ``$`` is treated as a reference (``$input.foo``)
+    and replaced with the resolved value. Any other value - including dicts
+    and lists, which are walked recursively - passes through literally.
+    """
+    if isinstance(template, str) and template.startswith('$'):
+        return resolve_ref(context, template[1:])
+    if isinstance(template, dict):
+        return {k: build_value(v, context) for k, v in template.items()}
+    if isinstance(template, list):
+        return [build_value(v, context) for v in template]
+    return template

evalkit/report.py

@@ -0,0 +1,186 @@
+"""Markdown rendering of scorecards and comparisons (CI / PR comments)."""
+
+from evalkit import models
+
+
+def _fmt(value: float | None) -> str:
+    return 'n/a' if value is None else f'{value:.4f}'
+
+
+def _fmt_delta(value: float | None) -> str:
+    return 'n/a' if value is None else f'{value:+.4f}'
+
+
+def render_scorecard(scorecard: models.Scorecard) -> str:
+    """Render a single scorecard as a Markdown section."""
+    lines = [
+        f'### {scorecard.project}/{scorecard.suite} '
+        f'- `{scorecard.variant.name}`',
+        '',
+        f'- model: `{scorecard.model_id}`  '
+        f'mode: `{scorecard.mode}`  '
+        f'dataset: `{scorecard.dataset_version}`  '
+        f'cases: {scorecard.n_cases}x{scorecard.n_samples}',
+        '',
+        '| metric | value |',
+        '| --- | ---: |',
+    ]
+    lines += [
+        f'| {m.metric} | {_fmt(m.value)} |' for m in scorecard.metrics.values()
+    ]
+    return '\n'.join(lines)
+
+
+_VERDICT_BADGE = {'pass': '**PASS**', 'warn': '**WARN**', 'fail': '**FAIL**'}
+
+
+def render_comparison(comparison: models.Comparison) -> str:
+    """Render a candidate-vs-baseline comparison as Markdown."""
+    badge = _VERDICT_BADGE.get(comparison.verdict, comparison.verdict)
+    lines = [
+        f'## {badge} - {comparison.project}/{comparison.suite}',
+        '',
+        f'`{comparison.candidate_variant}` vs '
+        f'`{comparison.baseline_variant}` - {comparison.summary}',
+        '',
+        '| metric | baseline | candidate | delta |',
+        '| --- | ---: | ---: | ---: |',
+    ]
+    for delta in comparison.deltas:
+        marker = (
+            f' **({comparison.win})**'
+            if delta.metric == comparison.win_metric
+            else ''
+        )
+        lines.append(
+            f'| {delta.metric}{marker} | {_fmt(delta.baseline)} '
+            f'| {_fmt(delta.candidate)} | {_fmt_delta(delta.delta)} |'
+        )
+    if comparison.guardrails:
+        lines += ['', '**Guardrails**', '']
+        for guard in comparison.guardrails:
+            mark = 'ok' if guard.passed else 'BREACH'
+            lines.append(f'- [{mark}] `{guard.metric}` - {guard.detail}')
+    return '\n'.join(lines)
+
+
+def render_agreement(result: models.AgreementResult) -> str:
+    """Render a judge<->human agreement result as Markdown."""
+    lines = [
+        f'### judge<->human agreement - `{result.judge_name}`',
+        '',
+        f'- {result.n_ratings} ratings from {result.n_raters} rater(s), '
+        f'scale 1..{result.scale}',
+        f'- overall: MAE {_fmt(result.overall_mae)}, '
+        f'r {_fmt(result.overall_correlation)}',
+        '',
+        '| dimension | n | human | judge | MAE | corr |',
+        '| --- | ---: | ---: | ---: | ---: | ---: |',
+    ]
+    for dim in result.dimensions:
+        lines.append(
+            f'| {dim.dimension} | {dim.n} | {_fmt(dim.human_mean)} '
+            f'| {_fmt(dim.judge_mean)} | {_fmt(dim.mae)} '
+            f'| {_fmt(dim.correlation)} |'
+        )
+    return '\n'.join(lines)
+
+
+def render_sweep(result: models.SweepResult) -> str:
+    """Render an N-way sweep as a ranking + a metric x variant leaderboard."""
+    variants = [c.variant for c in result.entries] or sorted(
+        {v for row in result.matrix.values() for v in row}
+    )
+    lines = [f'## sweep - {result.project}/{result.suite}', '']
+    if result.win_metric:
+        direction = 'higher' if result.win_higher_is_better else 'lower'
+        lines += [
+            f'ranked by `{result.win_metric}` ({direction} is better)',
+            '',
+            '| rank | variant | win |',
+            '| ---: | --- | ---: |',
+        ]
+        lines += [
+            f'| {e.rank} | `{e.variant}` | {_fmt(e.win_value)} |'
+            for e in result.entries
+        ]
+        lines.append('')
+
+    header = '| metric | ' + ' | '.join(variants) + ' |'
+    sep = '| --- |' + ' ---: |' * len(variants)
+    lines += ['**Leaderboard**', '', header, sep]
+    for metric, row in result.matrix.items():
+        present = [v for v in row.values() if v is not None]
+        best = max(present) if present else None  # display hint only
+        cells = []
+        for variant in variants:
+            value = row.get(variant)
+            text = _fmt(value)
+            if value is not None and best is not None and value == best:
+                text = f'**{text}**'
+            cells.append(text)
+        lines.append(f'| {metric} | ' + ' | '.join(cells) + ' |')
+    return '\n'.join(lines)
+
+
+def render_pairwise(result: models.PairwiseResult) -> str:
+    """Render an A-vs-B pairwise win-rate result as Markdown."""
+    rate = _fmt(result.win_rate_a)
+    return '\n'.join(
+        [
+            f'## pairwise - {result.project}/{result.suite}',
+            '',
+            f'`{result.variant_a}` (A) vs `{result.variant_b}` (B), '
+            f'judge `{result.judge_name}@{result.judge_version}`',
+            '',
+            f'- **A win-rate: {rate}** (ties = half) over {result.n} cases',
+            f'- A wins {result.a_wins} - B wins {result.b_wins} '
+            f'- ties {result.ties}',
+        ]
+    )
+
+
+def render_preferences(result: models.PreferenceResult) -> str:
+    """Render a human side-by-side win-rate (overall + per dimension)."""
+    lines = [
+        f'## human preferences - {result.project}/{result.suite}',
+        '',
+        f'`{result.variant_a}` (A) vs `{result.variant_b}` (B), '
+        f'{result.n} preferences from {result.n_raters} rater(s)',
+        '',
+        f'- **A win-rate: {_fmt(result.win_rate_a)}** (ties = half) '
+        f'- A wins {result.a_wins} - B wins {result.b_wins} '
+        f'- ties {result.ties}',
+        '',
+        '| dimension | n | A wins | B wins | ties | A win-rate |',
+        '| --- | ---: | ---: | ---: | ---: | ---: |',
+    ]
+    for dim in result.dimensions:
+        lines.append(
+            f'| {dim.dimension} | {dim.n} | {dim.a_wins} | {dim.b_wins} '
+            f'| {dim.ties} | {_fmt(dim.win_rate_a)} |'
+        )
+    return '\n'.join(lines)
+
+
+def render_pairwise_agreement(result: models.PairwiseAgreement) -> str:
+    """Render human-panel vs LLM-judge head-to-head agreement as Markdown."""
+    lines = [
+        f'## pairwise agreement - human vs `{result.judge_name}`',
+        '',
+        f'`{result.variant_a}` (A) vs `{result.variant_b}` (B)',
+        '',
+        f'- **agreement: {_fmt(result.agreement_rate)}** '
+        f'({result.agree}/{result.n} cases pick the same winner)',
+        f'- A win-rate: human {_fmt(result.human_win_rate_a)} '
+        f'- judge {_fmt(result.judge_win_rate_a)}',
+        '',
+        '| case | human | judge | agree |',
+        '| --- | :---: | :---: | :---: |',
+    ]
+    for case in result.outcomes:
+        mark = 'yes' if case.agree else 'NO'
+        lines.append(
+            f'| {case.case_id} | {case.human} | {case.judge} | {mark} |'
+        )
+    return '\n'.join(lines)

evalkit/reporters/__init__.py

@@ -0,0 +1,33 @@
+"""Report rendering, pluggable by ``type`` like adapters and graders.
+
+Importing this package registers the built-in ``markdown`` and ``html``
+reporters. Select one with :func:`build_reporter` (the CLI's ``--report``
+flag), and register more with ``@evalkit.reporters.base.register(...)``.
+"""
+
+from evalkit.reporters import html, markdown  # noqa: F401 - register built-ins
+from evalkit.reporters.base import (
+    Reporter,
+    build_reporter,
+    per_case_matrix,
+    register,
+    render_agreement,
+    render_pairwise_agreement,
+    render_preferences,
+    render_run,
+    run_body,
+    wrap_document,
+)
+
+__all__ = [
+    'Reporter',
+    'build_reporter',
+    'per_case_matrix',
+    'register',
+    'render_agreement',
+    'render_pairwise_agreement',
+    'render_preferences',
+    'render_run',
+    'run_body',
+    'wrap_document',
+]

evalkit/reporters/base.py

@@ -0,0 +1,159 @@
+"""Reporter protocol and registry (the report-rendering seam).
+
+A reporter turns engine results into a rendered report. Two report kinds are
+first-class, mirroring the two analyses the engine produces:
+
+* ``scorecard`` - a SINGLE analysis (one suite x one variant run);
+* ``comparison`` - a COMPARATIVE analysis (candidate vs baseline).
+
+Built-ins ``markdown`` and ``html`` cover both. Register more the same way as
+adapters and graders - ``@reporters.base.register('pdf')`` + ``--plugins`` -
+so a consumer can add a format without touching the core.
+
+Three further kinds are optional; a reporter is discovered to support each
+via ``getattr`` and only implements the ones it cares about:
+
+* ``run`` - a scorecard plus its per-case drill-down (see :func:`render_run`);
+* ``agreement`` - a judge<->human calibration table (see
+  :func:`render_agreement`);
+* ``document`` - wrap a rendered body in a standalone document (e.g. an HTML
+  page). Reporters that omit it are treated as identity.
+
+Every ``*`` method returns a *fragment*; wrapping into a standalone document
+happens once, at the edge, so fragments compose (a ``run`` body and an
+``agreement`` table can share one page).
+"""
+
+import typing
+
+from evalkit import models
+
+
+@typing.runtime_checkable
+class Reporter(typing.Protocol):
+    """Render a single scorecard and a candidate-vs-baseline comparison."""
+
+    def scorecard(self, scorecard: models.Scorecard) -> str: ...
+
+    def comparison(self, comparison: models.Comparison) -> str: ...
+
+
+_REGISTRY: dict[str, type] = {}
+
+
+def register(type_name: str) -> typing.Callable[[type], type]:
+    """Class decorator registering a reporter under a report ``type``."""
+
+    def _decorate(cls: type) -> type:
+        if type_name in _REGISTRY:
+            raise ValueError(f'report type {type_name!r} already registered')
+        _REGISTRY[type_name] = cls
+        return cls
+
+    return _decorate
+
+
+def build_reporter(spec: str | dict) -> Reporter:
+    """Instantiate a reporter from a name or a ``{type, ...}`` config spec."""
+    spec = {'type': spec} if isinstance(spec, str) else dict(spec)
+    type_name = spec.pop('type')
+    if type_name not in _REGISTRY:
+        raise ValueError(
+            f'unknown report type {type_name!r}; known: {sorted(_REGISTRY)}'
+        )
+    return _REGISTRY[type_name](**spec)
+
+
+def wrap_document(reporter: Reporter, body: str) -> str:
+    """Wrap ``body`` with the reporter's optional ``document`` hook."""
+    document = getattr(reporter, 'document', None)
+    return document(body) if callable(document) else body
+
+
+def per_case_matrix(run: models.RunResult) -> tuple[list[str], list[dict]]:
+    """Per-case drill-down behind a scorecard's aggregate means.
+
+    Returns ``(metric_names, rows)`` where each row is
+    ``{label, error, output, cells}`` - ``cells`` maps each per-case metric
+    to its :class:`~evalkit.models.Score` (or is absent). Reporters format
+    this into a case x metric table so a low aggregate points at the case
+    that caused it.
+    """
+    metrics = sorted(
+        {
+            score.metric
+            for result in run.results
+            for score in result.scores
+            if score.kind == 'per_case'
+        }
+    )
+    multi = run.scorecard.n_samples > 1
+    rows: list[dict] = []
+    for result in run.results:
+        by_metric = {
+            s.metric: s for s in result.scores if s.kind == 'per_case'
+        }
+        label = result.case.id + (f' #{result.sample_idx}' if multi else '')
+        rows.append(
+            {
+                'label': label,
+                'error': result.output.error,
+                'output': result.output,
+                'cells': by_metric,
+            }
+        )
+    return metrics, rows
+
+
+def render_run(reporter: Reporter, run: models.RunResult) -> str:
+    """Render a full run as a standalone document: the detailed per-case
+    report if the reporter supports ``run``, else the aggregate scorecard."""
+    return wrap_document(reporter, run_body(reporter, run))
+
+
+def run_body(reporter: Reporter, run: models.RunResult) -> str:
+    """The run's report *fragment* (unwrapped), so it can be composed with
+    other fragments before a single :func:`wrap_document` at the edge."""
+    detailed = getattr(reporter, 'run', None)
+    if callable(detailed):
+        return detailed(run)
+    return reporter.scorecard(run.scorecard)
+
+
+def render_agreement(
+    reporter: Reporter, result: models.AgreementResult
+) -> str:
+    """A judge<->human agreement *fragment*, using the reporter's ``agreement``
+    hook when present and falling back to the Markdown table otherwise."""
+    render = getattr(reporter, 'agreement', None)
+    if callable(render):
+        return render(result)
+    from evalkit import report
+
+    return report.render_agreement(result)
+
+
+def render_preferences(
+    reporter: Reporter, result: models.PreferenceResult
+) -> str:
+    """A human A-vs-B win-rate *fragment*, using the reporter's ``preferences``
+    hook when present and falling back to the Markdown table otherwise."""
+    render = getattr(reporter, 'preferences', None)
+    if callable(render):
+        return render(result)
+    from evalkit import report
+
+    return report.render_preferences(result)
+
+
+def render_pairwise_agreement(
+    reporter: Reporter, result: models.PairwiseAgreement
+) -> str:
+    """A human-panel vs LLM-judge agreement *fragment*, using the reporter's
+    ``pairwise_agreement`` hook when present, else the Markdown table."""
+    render = getattr(reporter, 'pairwise_agreement', None)
+    if callable(render):
+        return render(result)
+    from evalkit import report
+
+    return report.render_pairwise_agreement(result)