evalcore 2.3.0__tar.gz → 2.4.0__tar.gz

{evalcore-2.3.0 → evalcore-2.4.0}/CHANGELOG.md

@@ -6,6 +6,34 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.4.0] - 2026-08-09
+
+A run without a baseline can now say whether it passed.
+
+### Added
+- `RunResult.checks`, a `models.ThresholdCheck` the runner fills in: the
+  suite's **absolute** guardrails (`min`/`max`) measured against this run.
+  Most of a suite's rules are absolute and answerable from one run; only the
+  win metric and the relative rules need a baseline. Nothing new to call -
+  `compare` is for comparing, and a single run was never a comparison.
+- `compare.check_thresholds(scorecard, thresholds)`, which the runner uses.
+
+### Changed
+- **Breaking:** `GuardrailResult.passed` is now `bool | None`. `None` means
+  the rule could not be evaluated - a `must_not_increase` /
+  `must_not_decrease` with no baseline. Test `passed is False` for a breach;
+  `not passed` now also catches the skipped case. The Markdown and HTML
+  reporters render it as `skipped` rather than a breach.
+- `store.score_rows` reports `run.checks` on the gate columns when no
+  `Comparison` is passed: `gate_verdict` and the per-metric `guardrail` become
+  real, while `gate_win` stays `'none'`. That is deliberate - `gate_win` is
+  what marks the three `win_*` fields as never computed rather than measured
+  at zero, and an ungated run must not claim a comparison happened. A rule
+  that could not run reports `guardrail = 'none'` with the reason in
+  `guardrail_gap`, not a false `pass`.
+- A `Comparison` supersedes the run's own checks: it evaluates the same rules
+  with a baseline available, so it gets the relative ones too.
+
 ## [2.3.0] - 2026-08-08
 
 A run now describes its own scores.
@@ -220,7 +248,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.3.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.0...HEAD
+[2.4.0]: https://github.com/scottpmiller/evalcore/compare/2.3.0...2.4.0
 [2.3.0]: https://github.com/scottpmiller/evalcore/compare/2.2.0...2.3.0
 [2.2.0]: https://github.com/scottpmiller/evalcore/compare/2.1.0...2.2.0
 [2.1.0]: https://github.com/scottpmiller/evalcore/compare/2.0.0...2.1.0

{evalcore-2.3.0 → evalcore-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.3.0
+Version: 2.4.0
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.3.0 → evalcore-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.3.0"
+version = "2.4.0"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/compare.py

@@ -23,11 +23,27 @@ def _metric(card: models.Scorecard, name: str) -> float | None:
 
 
 def _check_guardrail(
-    rule: dict, baseline: models.Scorecard, candidate: models.Scorecard
+    rule: dict, baseline: models.Scorecard | None, candidate: models.Scorecard
 ) -> models.GuardrailResult:
+    """Evaluate one guardrail rule.
+
+    ``baseline`` is ``None`` when only one run exists. The absolute parts of
+    a rule (``min``/``max``) still evaluate; the relative parts
+    (``must_not_increase``/``must_not_decrease``) cannot, and are reported
+    rather than silently passed - a rule with nothing left to check comes
+    back ``passed=None``.
+    """
     metric = rule['metric']
     cand = _metric(candidate, metric)
-    base = _metric(baseline, metric)
+    base = _metric(baseline, metric) if baseline is not None else None
+    relative = [
+        name
+        for name in ('must_not_increase', 'must_not_decrease')
+        if rule.get(name)
+    ]
+    # A relative rule with no baseline is unevaluable, not satisfied.
+    skipped = relative if baseline is None else []
+    absolute = 'max' in rule or 'min' in rule
     if cand is None:
         return models.GuardrailResult(
             metric=metric, passed=False, detail='metric absent on candidate'
@@ -51,12 +67,54 @@ def _check_guardrail(
     ):
         problems.append(f'decreased {base:.4f} -> {cand:.4f}')
 
+    note = (
+        f'{", ".join(skipped)} not evaluated: needs a baseline'
+        if skipped
+        else ''
+    )
     if problems:
+        detail = '; '.join([*problems, note] if note else problems)
         return models.GuardrailResult(
-            metric=metric, passed=False, detail='; '.join(problems)
+            metric=metric, passed=False, detail=detail
         )
-    return models.GuardrailResult(
-        metric=metric, passed=True, detail=f'{cand:.4f} ok'
+    if skipped and not absolute:
+        # Nothing was checked, so this is neither a pass nor a breach.
+        return models.GuardrailResult(metric=metric, passed=None, detail=note)
+    detail = f'{cand:.4f} ok' + (f'; {note}' if note else '')
+    return models.GuardrailResult(metric=metric, passed=True, detail=detail)
+
+
+def check_thresholds(
+    scorecard: models.Scorecard, thresholds: dict | None = None
+) -> models.ThresholdCheck:
+    """Measure one run against its suite's absolute thresholds.
+
+    A gate needs two runs, but a suite's guardrails are mostly absolute - a
+    ceiling on the error rate, a floor on a format check - and those are
+    answerable from a single run. The runner calls this so a run carries its
+    own verdict; a caller with one run should not have to reach for
+    ``compare``, which is for comparing.
+
+    The win metric is deliberately not evaluated: it is a comparison by
+    definition, and reporting it here would claim a baseline that does not
+    exist. Relative guardrails come back ``passed=None`` for the same reason.
+
+    Args:
+        scorecard: The run to measure.
+        thresholds: The suite's ``thresholds`` block.
+
+    Returns:
+        The verdict and one result per configured guardrail. ``verdict`` is
+        ``'none'`` when the suite declares no guardrails.
+
+    """
+    rules = (thresholds or {}).get('guardrails', [])
+    if not rules:
+        return models.ThresholdCheck()
+    guardrails = [_check_guardrail(rule, None, scorecard) for rule in rules]
+    breached = [g for g in guardrails if g.passed is False]
+    return models.ThresholdCheck(
+        verdict='fail' if breached else 'pass', guardrails=guardrails
     )
 
 
@@ -105,7 +163,7 @@ def compare(
     ]
     win_metric, win = _evaluate_win(thresholds, baseline, candidate)
 
-    breached = [g for g in guardrails if not g.passed]
+    breached = [g for g in guardrails if g.passed is False]
     on_regression = thresholds.get('on_regression', 'warn')
     if breached:
         verdict = 'fail'

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/models.py

@@ -184,6 +184,38 @@ class GraderInfo(pydantic.BaseModel):
     scale: int = 0
 
 
+class GuardrailResult(pydantic.BaseModel):
+    """Outcome of one guardrail check against the candidate.
+
+    ``passed`` is tri-state. ``None`` means the rule could not be evaluated -
+    a ``must_not_increase`` / ``must_not_decrease`` rule with no baseline to
+    compare against. That is neither a pass nor a breach, and conflating it
+    with either claims a check that never ran, so callers test
+    ``passed is False`` for a breach rather than ``not passed``.
+    """
+
+    metric: str
+    passed: bool | None
+    detail: str
+
+
+class ThresholdCheck(pydantic.BaseModel):
+    """A run measured against its suite's absolute thresholds.
+
+    A gate needs two runs, but most of a suite's guardrails are absolute -
+    a ceiling on the error rate, a floor on a format check - and those are
+    answerable from one run. This is that answer, computed by the runner so
+    a single run can say whether it passed without a baseline and without
+    the caller re-deriving anything.
+
+    ``verdict`` is ``'none'`` when the suite declares no thresholds. The win
+    metric is never evaluated here: it is a comparison by definition.
+    """
+
+    verdict: typing.Literal['none', 'pass', 'warn', 'fail'] = 'none'
+    guardrails: list[GuardrailResult] = pydantic.Field(default_factory=list)
+
+
 class RunResult(pydantic.BaseModel):
     """Everything one run produced: the scorecard plus per-sample results.
 
@@ -208,6 +240,7 @@ class RunResult(pydantic.BaseModel):
     results: list[CaseResult] = pydantic.Field(default_factory=list)
     aggregate_scores: list[Score] = pydantic.Field(default_factory=list)
     graders: dict[str, GraderInfo] = pydantic.Field(default_factory=dict)
+    checks: ThresholdCheck = pydantic.Field(default_factory=ThresholdCheck)
 
 
 class Rating(pydantic.BaseModel):
@@ -421,14 +454,6 @@ class MetricDelta(pydantic.BaseModel):
     delta: float | None
 
 
-class GuardrailResult(pydantic.BaseModel):
-    """Outcome of one guardrail check against the candidate."""
-
-    metric: str
-    passed: bool
-    detail: str
-
-
 class Comparison(pydantic.BaseModel):
     """Candidate-vs-baseline comparison and gate verdict."""
 

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/report.py

@@ -59,7 +59,15 @@ def render_comparison(comparison: models.Comparison) -> str:
     if comparison.guardrails:
         lines += ['', '**Guardrails**', '']
         for guard in comparison.guardrails:
-            mark = 'ok' if guard.passed else 'BREACH'
+            # Tri-state: None is a rule that could not run (a relative
+            # guardrail with no baseline), which is not a breach.
+            mark = (
+                'skipped'
+                if guard.passed is None
+                else 'ok'
+                if guard.passed
+                else 'BREACH'
+            )
             lines.append(f'- [{mark}] `{guard.metric}` - {guard.detail}')
     return '\n'.join(lines)
 

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/reporters/html.py

@@ -158,6 +158,7 @@ _CSS = (
     '.guards li{padding:4px 0;font-size:14px}'
     '.guards li.ok::before{content:"\\2713 ";color:#1f8f5f}'
     '.guards li.breach::before{content:"\\2717 ";color:#c0392b}'
+    '.guards li.skipped::before{content:"\\2013 ";color:#8a8a8a}'
     'td.fail{color:#c0392b;font-weight:600}'
     'details{margin:6px 0}summary{cursor:pointer;color:#68707a;font-size:13px}'
     'pre{background:#f0f2f4;border-radius:6px;padding:10px;overflow:auto;'
@@ -196,6 +197,17 @@ def _esc(value) -> str:
     return _html.escape(str(value))
 
 
+def _guard_class(guard) -> str:
+    """CSS class for one guardrail result.
+
+    ``passed`` is tri-state: ``None`` is a rule that could not run - a
+    relative guardrail with no baseline - which must not render as a breach.
+    """
+    if guard.passed is None:
+        return 'skipped'
+    return 'ok' if guard.passed else 'breach'
+
+
 @base.register('html')
 class HtmlReporter:
     """Render scorecards and comparisons as self-contained HTML."""
@@ -247,7 +259,7 @@ class HtmlReporter:
         guards = ''
         if comparison.guardrails:
             items = ''.join(
-                f'<li class="{"ok" if g.passed else "breach"}">'
+                f'<li class="{_guard_class(g)}">'
                 f'<code>{_esc(g.metric)}</code> &mdash; {_esc(g.detail)}</li>'
                 for g in comparison.guardrails
             )

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/runner.py

@@ -18,7 +18,7 @@ import statistics
 import time
 import uuid
 
-from evalcore import loader, models, store
+from evalcore import compare, loader, models, store
 from evalcore import retry as retry_mod
 from evalcore.adapters import base as adapters_base
 from evalcore.graders import base as graders_base
@@ -351,6 +351,10 @@ async def run_suite(
         results=results,
         aggregate_scores=agg_scores,
         graders=_grader_info(per_case_graders, aggregate_graders),
+        # So a single run can say whether it passed. Absolute rules only -
+        # the win metric and the relative guardrails need a baseline, and a
+        # gate's Comparison supersedes this when there is one.
+        checks=compare.check_thresholds(scorecard, suite.thresholds),
     )
 
 

{evalcore-2.3.0 → evalcore-2.4.0}/src/evalcore/store.py

@@ -287,7 +287,9 @@ def _run_key(scorecard: models.Scorecard) -> dict:
 
 
 def _gate(
-    comparison: models.Comparison | None, baseline_run_id: str | None
+    comparison: models.Comparison | None,
+    baseline_run_id: str | None,
+    checks: models.ThresholdCheck | None = None,
 ) -> dict:
     """The run-grain gate columns, repeated on every row of a gated run.
 
@@ -297,15 +299,26 @@ def _gate(
     0 delta genuinely means no change.
     """
     if comparison is None:
+        # Without a Comparison the run may still have been measured against
+        # its suite's absolute thresholds. That is a real verdict, so it is
+        # reported - but `gate_win` stays 'none', which is what marks the
+        # three win_* fields as never computed rather than measured at zero.
+        checks = checks or models.ThresholdCheck()
+        breached = [g for g in checks.guardrails if g.passed is False]
         return {
-            'gate_verdict': 'none',
+            'gate_verdict': checks.verdict,
             'gate_win': 'none',
             'baseline_run_id': _NO_UUID,
             'baseline_variant': '',
             'win_baseline': None,
             'win_candidate': None,
             'win_delta': None,
-            'gate_summary': '',
+            'gate_summary': (
+                'guardrail breach: '
+                + '; '.join(f'{g.metric} ({g.detail})' for g in breached)
+                if breached
+                else ''
+            ),
         }
     delta = next(
         (d for d in comparison.deltas if d.metric == comparison.win_metric),
@@ -332,8 +345,14 @@ def _metric_gate(
     rail = rails.get(metric)
     return {
         'win': bool(comparison and comparison.win_metric == metric),
+        # rail.passed is tri-state: None means the rule needed a baseline
+        # and could not run, which reads as 'none' rather than a false pass.
         'guardrail': (
-            'none' if rail is None else 'pass' if rail.passed else 'fail'
+            'none'
+            if rail is None or rail.passed is None
+            else 'pass'
+            if rail.passed
+            else 'fail'
         ),
         'guardrail_gap': rail.detail if rail else '',
     }
@@ -410,12 +429,15 @@ def score_rows(
     :func:`grader_lookups` builds them from the suite.
     """
     key = _run_key(run.scorecard)
-    gate = _gate(comparison, baseline_run_id)
-    rails = (
-        {rail.metric: rail for rail in comparison.guardrails}
-        if comparison
-        else {}
-    )
+    gate = _gate(comparison, baseline_run_id, run.checks)
+    # A Comparison supersedes the run's own checks: it evaluates the same
+    # rules with a baseline available, so it gets the relative ones too.
+    rails = {
+        rail.metric: rail
+        for rail in (
+            comparison.guardrails if comparison else run.checks.guardrails
+        )
+    }
     # The run describes its own graders as of 2.3.0. The lookups remain for
     # runs written before that, which carry an empty map - a `run.json` on
     # disk outlives the release that wrote it. An explicit lookup still wins,

{evalcore-2.3.0 → evalcore-2.4.0}/tests/test_store.py

@@ -3,9 +3,10 @@
 import json
 import pathlib
 import tempfile
+import typing
 import unittest
 
-from evalcore import errors, graders, models, store
+from evalcore import compare, errors, graders, models, store
 
 
 def _run(with_failure: bool = False):
@@ -394,6 +395,77 @@ class SelfDescribingRunTests(unittest.TestCase):
         self.assertEqual(types['j'], 'llm_as_judge')
 
 
+class UngatedVerdictTests(unittest.TestCase):
+    """A run with no baseline still reports its absolute thresholds."""
+
+    THRESHOLDS: typing.ClassVar = {
+        'guardrails': [
+            # absolute, breaches: f1 is 0.9
+            {'metric': 'f1', 'min': 1.0},
+            # absolute, passes
+            {'metric': 'passed_check', 'max': 2.0},
+            # relative only: unevaluable without a baseline
+            {'metric': 'quality.overall', 'must_not_decrease': True},
+        ]
+    }
+
+    def _rows(self):
+        run = _run()
+        # A real value, so the relative rule is unevaluable rather than
+        # absent - an absent metric is a breach and outranks the skip.
+        run.scorecard.metrics['quality.overall'] = models.MetricValue(
+            metric='quality.overall', value=0.8, kind='mean', n=2
+        )
+        run.checks = compare.check_thresholds(run.scorecard, self.THRESHOLDS)
+        return run, store.score_rows(run)
+
+    def test_the_verdict_is_real(self):
+        run, rows = self._rows()
+        self.assertEqual(run.checks.verdict, 'fail')  # f1 0.9 < min 1.0
+        self.assertEqual(rows[0]['gate_verdict'], 'fail')
+
+    def test_the_win_columns_stay_uncomputed(self):
+        """gate_win is what marks the three zeros as never measured."""
+        _, rows = self._rows()
+        self.assertEqual(rows[0]['gate_win'], 'none')
+        self.assertIsNone(rows[0]['win_baseline'])
+        self.assertIsNone(rows[0]['win_delta'])
+        self.assertEqual(rows[0]['baseline_run_id'], store._NO_UUID)
+
+    def test_an_absolute_breach_lands_on_its_metric(self):
+        _, rows = self._rows()
+        rails = {r['metric']: r['guardrail'] for r in rows}
+        self.assertEqual(rails['f1'], 'fail')
+
+    def test_a_relative_rule_is_not_a_pass(self):
+        """It could not run, so it reads 'none' with the reason in the gap."""
+        _, rows = self._rows()
+        row = next(r for r in rows if r['metric'] == 'quality.overall')
+        self.assertEqual(row['guardrail'], 'none')
+        self.assertIn('needs a baseline', row['guardrail_gap'])
+
+    def test_a_comparison_supersedes_the_runs_own_checks(self):
+        run, _ = self._rows()
+        rows = store.score_rows(run, _comparison(), baseline_run_id='B')
+        self.assertEqual(rows[0]['gate_verdict'], 'fail')
+        self.assertNotEqual(rows[0]['gate_win'], 'none')
+        self.assertIsNotNone(rows[0]['win_delta'])
+
+    def test_an_absent_metric_outranks_the_skip(self):
+        """Nothing to measure is a breach, not an unevaluated rule."""
+        run = _run()
+        check = compare.check_thresholds(
+            run.scorecard, {'guardrails': [{'metric': 'nope', 'min': 1.0}]}
+        )
+        self.assertIs(check.guardrails[0].passed, False)
+        self.assertIn('absent', check.guardrails[0].detail)
+
+    def test_no_thresholds_means_no_verdict(self):
+        run = _run()
+        self.assertEqual(run.checks.verdict, 'none')
+        self.assertEqual(store.score_rows(run)[0]['gate_verdict'], 'none')
+
+
 class ScoreExporterProtocolTests(unittest.TestCase):
     """The seam another package implements to publish to a real store."""
 

{evalcore-2.3.0 → evalcore-2.4.0}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.3.0"
+version = "2.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },