evalcore 2.6.0__tar.gz → 2.6.1__tar.gz

{evalcore-2.6.0 → evalcore-2.6.1}/CHANGELOG.md

@@ -6,6 +6,45 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.6.1] - 2026-09-12
+
+The facts 2.6.0 added reach a results store.
+
+Shipped as a patch. It changes what published rows contain, so the 1.0.0
+policy would call it a minor; it goes out as 2.6.1 as a deliberate exception,
+because it completes 2.6.0 rather than adding to it. Same call 2.4.2 made, for
+the same reason.
+
+### Added
+- `score_rows` exports the direction and range 2.6.0 introduced. That release
+  gave every metric both and then kept them in memory: the row shape this
+  module flattens a run into carried neither, so nothing reached a results
+  store and no consumer downstream could use them. Five columns on every row
+  shape, including the failed-invocation row that names no metric at all.
+  - `value_range` (`none` / `bounded` / `unbounded_above` /
+    `unbounded_below` / `unbounded`), `value_minimum` and `value_maximum`.
+    The discriminator is what makes "declared unbounded" and "never declared"
+    different answers, which two nullable floats cannot express; it is the
+    idiom `metric_kind = 'none'` already uses beside a filled-in `value`. All
+    five members are spelled out because `MetricRange` can express all five -
+    `range: {max: 5}` on a numeric field produces `unbounded_below` - and a
+    consumer storing this as an enum pays a migration to add a member later.
+  - `metric_direction` and `higher_is_better`. `metric_direction` carries the
+    same four values `gate_win` does, `none` among them, meaning never
+    computed: an ungated run has no baseline, and `neutral` would report
+    "measured, and it did not move" about a number nothing was measured
+    against. `higher_is_better` is `'true'|'false'|'unknown'`, and `unknown`
+    is narrower than it looks - `compare()` defaults a metric nothing
+    mentions to higher-is-better, so it reads `unknown` only when a suite
+    declared the metric neutral or there was no comparison at all.
+
+### Changed
+- `_passed` is now `_tristate`, taking the spelling of its third state as an
+  argument. `passed` keeps `'null'`, which it has published since the row
+  shape existed; `higher_is_better` uses `'unknown'`, which reads as "nobody
+  declared one" rather than as a missing value. Private; no caller outside
+  this module.
+
 ## [2.6.0] - 2026-09-12
 
 A metric says which way is good and what its values sit on.
@@ -431,7 +470,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.6.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.6.1...HEAD
+[2.6.1]: https://github.com/scottpmiller/evalcore/compare/2.6.0...2.6.1
 [2.6.0]: https://github.com/scottpmiller/evalcore/compare/2.5.0...2.6.0
 [2.5.0]: https://github.com/scottpmiller/evalcore/compare/2.4.3...2.5.0
 [2.4.3]: https://github.com/scottpmiller/evalcore/compare/2.4.2...2.4.3

{evalcore-2.6.0 → evalcore-2.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalcore
-Version: 2.6.0
+Version: 2.6.1
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.6.0 → evalcore-2.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.6.0"
+version = "2.6.1"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.6.0 → evalcore-2.6.1}/src/evalcore/store.py

@@ -259,6 +259,13 @@ _NO_JUDGES = {
     'judge_scale': 0,
 }
 
+#: A metric whose grader declared no range. ``value_range`` is the
+#: discriminator, in the same idiom as ``metric_kind = 'none'`` sitting beside
+#: a filled-in ``value``: the two floats here are padding, not measurements,
+#: and a consumer that reads them without the discriminator learns that every
+#: undeclared metric runs 0..0.
+_NO_RANGE = {'value_range': 'none', 'value_minimum': 0.0, 'value_maximum': 0.0}
+
 
 def _run_key(scorecard: models.Scorecard) -> dict:
     """The reproducibility key carried on every store row.
@@ -336,13 +343,63 @@ def _gate(
     }
 
 
+def _metric_range(value_range: models.MetricRange | None) -> dict:
+    """What the metric's values sit on, as three null-free columns.
+
+    ``value_range`` says which of the two floats mean anything, and covers
+    every shape a ``MetricRange`` can take rather than the shapes anything
+    currently emits:
+
+    ``none``
+        No range was declared at all. Render the raw number.
+    ``bounded``
+        Both ends known. ``0..1`` is the case that makes a percentage safe.
+    ``unbounded_above``
+        ``value_minimum`` holds, no ceiling. A count, a cost, an elapsed
+        time - not a fraction of anything.
+    ``unbounded_below``
+        ``value_maximum`` holds, no floor.
+    ``unbounded``
+        A range object carrying neither end, which says less than declaring
+        nothing but is still a different thing from it.
+
+    The discriminator is why this is not two nullable floats: "declared
+    unbounded" and "never declared" are different answers, and a null in
+    both columns cannot tell them apart. The two open-ended members exist
+    because ``MetricRange`` can express them - ``range: {max: 5}`` on a
+    numeric field gives exactly the third case - and because a consumer
+    storing this as an enum pays an ALTER to add a member later, so the
+    degenerate cases are cheaper spelled out now than discovered in
+    production.
+    """
+    if value_range is None:
+        return dict(_NO_RANGE)
+    low, high = value_range.minimum, value_range.maximum
+    kind = (
+        'bounded'
+        if low is not None and high is not None
+        else 'unbounded_above'
+        if low is not None
+        else 'unbounded_below'
+        if high is not None
+        else 'unbounded'
+    )
+    return {
+        'value_range': kind,
+        'value_minimum': float(low) if low is not None else 0.0,
+        'value_maximum': float(high) if high is not None else 0.0,
+    }
+
+
 def _metric_gate(
     metric: str,
     comparison: models.Comparison | None,
     rails: dict[str, models.GuardrailResult],
+    deltas: dict[str, models.MetricDelta] | None = None,
 ) -> dict:
     """The gate columns that belong to one metric rather than to the run."""
     rail = rails.get(metric)
+    delta = (deltas or {}).get(metric)
     return {
         'win': bool(comparison and comparison.win_metric == metric),
         # rail.passed is tri-state: None means the rule needed a baseline
@@ -355,6 +412,20 @@ def _metric_gate(
             else 'fail'
         ),
         'guardrail_gap': rail.detail if rail else '',
+        # 'none' is the fourth state the model does not have, and means the
+        # same thing gate_win = 'none' does at run grain: never computed. A
+        # direction is a comparison, so an ungated run cannot have one, and
+        # writing 'neutral' there would report "measured, and it did not
+        # move" for a number nothing was measured against.
+        'metric_direction': delta.direction if delta else 'none',
+        # Which way is good, as a tri-state string for the same reason
+        # `passed` is one: 'unknown' is a real answer, not a missing value.
+        # It is narrower than it looks - compare() defaults a metric nothing
+        # mentions to higher-is-better, so this only reads 'unknown' when a
+        # suite declared the metric neutral or there was no comparison.
+        'higher_is_better': _tristate(
+            delta.higher_is_better if delta else None, absent='unknown'
+        ),
     }
 
 
@@ -393,14 +464,19 @@ def _judges(score: models.Score, scale: int) -> dict:
     }
 
 
-def _passed(value: bool | None) -> str:
-    """``passed`` as a tri-state string: deterministic true/false, judge null.
+def _tristate(value: bool | None, absent: str = 'null') -> str:
+    """A three-state bool as a string, for a table with no Nullable columns.
 
-    A string rather than a bool-or-null because it is not a missing
-    measurement: a judge has no pass line by design, so 'null' is one of three
-    real states.
+    The third state is a real answer rather than a missing measurement, and
+    both callers mean something by it: a judge has no pass line by design,
+    and a metric the suite declared neutral has no good direction by design.
+
+    They spell it differently because their columns do. ``passed`` has said
+    ``'null'`` since the row shape was first published, and a store's
+    polarity column reads better as ``'unknown'`` - "nobody declared one" -
+    where ``'null'`` would look like a missing value.
     """
-    return 'true' if value is True else 'false' if value is False else 'null'
+    return 'true' if value is True else 'false' if value is False else absent
 
 
 def score_rows(
@@ -438,6 +514,12 @@ def score_rows(
             comparison.guardrails if comparison else run.checks.guardrails
         )
     }
+    # Keyed once rather than scanned per row: a run is (cases x samples x
+    # metrics) rows against one list of deltas.
+    deltas = {
+        delta.metric: delta
+        for delta in (comparison.deltas if comparison else [])
+    }
     # The run describes its own graders as of 2.3.0. The lookups remain for
     # runs written before that, which carry an empty map - a `run.json` on
     # disk outlives the release that wrote it. An explicit lookup still wins,
@@ -467,9 +549,16 @@ def score_rows(
                     'passed': 'null',
                     'detail': '',
                     'case_labels': result.case.labels,
+                    **_NO_RANGE,
                     'win': False,
                     'guardrail': 'none',
                     'guardrail_gap': '',
+                    # This row names no metric at all, so there is nothing to
+                    # have a range or a direction about. Spelled out rather
+                    # than routed through _metric_gate, which would look the
+                    # empty string up.
+                    'metric_direction': 'none',
+                    'higher_is_better': 'unknown',
                     **invocation,
                     **_NO_JUDGES,
                     **gate,
@@ -488,10 +577,11 @@ def score_rows(
                     'metric': score.metric,
                     'metric_kind': score.kind,
                     'value': score.value,
-                    'passed': _passed(score.passed),
+                    'passed': _tristate(score.passed),
                     'detail': score.detail or '',
                     'case_labels': result.case.labels,
-                    **_metric_gate(score.metric, comparison, rails),
+                    **_metric_range(score.value_range),
+                    **_metric_gate(score.metric, comparison, rails, deltas),
                     **invocation,
                     **_judges(score, scales.get(score.grader, 0)),
                     **gate,
@@ -510,10 +600,11 @@ def score_rows(
                 'metric': score.metric,
                 'metric_kind': 'aggregate',
                 'value': score.value,
-                'passed': _passed(score.passed),
+                'passed': _tristate(score.passed),
                 'detail': score.detail or '',
                 'case_labels': {},
-                **_metric_gate(score.metric, comparison, rails),
+                **_metric_range(score.value_range),
+                **_metric_gate(score.metric, comparison, rails, deltas),
                 **_NO_INVOCATION,
                 **_NO_JUDGES,
                 **gate,
@@ -537,7 +628,8 @@ def score_rows(
                 'passed': 'null',
                 'detail': '',
                 'case_labels': {},
-                **_metric_gate(metric, comparison, rails),
+                **_NO_RANGE,
+                **_metric_gate(metric, comparison, rails, deltas),
                 **_NO_INVOCATION,
                 **_NO_JUDGES,
                 **gate,

{evalcore-2.6.0 → evalcore-2.6.1}/tests/test_store.py

@@ -104,7 +104,12 @@ def _comparison():
         verdict='fail',
         deltas=[
             models.MetricDelta(
-                metric='passed_check', baseline=0.5, candidate=1.0, delta=0.5
+                metric='passed_check',
+                baseline=0.5,
+                candidate=1.0,
+                delta=0.5,
+                higher_is_better=True,
+                direction='improved',
             )
         ],
         guardrails=[
@@ -293,6 +298,98 @@ class RowTests(unittest.TestCase):
         self.assertEqual(row['guardrail_gap'], 'metric absent on candidate')
 
 
+class MetricFactRowTests(unittest.TestCase):
+    """Range and direction reach the row shape, or say they did not."""
+
+    def _rows(self, comparison=None):
+        return {
+            row['metric']: row for row in store.score_rows(_run(), comparison)
+        }
+
+    def _scored(self, value_range):
+        """A run whose one per-case score declares ``value_range``."""
+        run = _run()
+        run.results[0].scores[0].value_range = value_range
+        return {row['metric']: row for row in store.score_rows(run)}[
+            run.results[0].scores[0].metric
+        ]
+
+    def test_bounded_range_reaches_the_row(self):
+        row = self._scored(models.MetricRange(minimum=0.0, maximum=1.0))
+        self.assertEqual(row['value_range'], 'bounded')
+        self.assertEqual(row['value_minimum'], 0.0)
+        self.assertEqual(row['value_maximum'], 1.0)
+
+    def test_unbounded_is_a_different_answer_from_undeclared(self):
+        # The distinction two nullable floats could not carry: a count with
+        # no ceiling versus a metric nobody described.
+        unbounded = self._scored(models.MetricRange(minimum=0.0, maximum=None))
+        undeclared = self._scored(None)
+        self.assertEqual(unbounded['value_range'], 'unbounded_above')
+        self.assertEqual(undeclared['value_range'], 'none')
+        self.assertEqual(
+            unbounded['value_minimum'], undeclared['value_minimum']
+        )
+        self.assertEqual(
+            unbounded['value_maximum'], undeclared['value_maximum']
+        )
+
+    def test_a_range_open_at_the_bottom_keeps_the_end_it_has(self):
+        # `range: {max: 5}` on a numeric field produces exactly this, and
+        # collapsing it to 'none' would throw away a declared ceiling.
+        row = self._scored(models.MetricRange(minimum=None, maximum=5.0))
+        self.assertEqual(row['value_range'], 'unbounded_below')
+        self.assertEqual(row['value_maximum'], 5.0)
+        self.assertEqual(row['value_minimum'], 0.0)
+
+    def test_a_range_carrying_neither_end_is_not_the_same_as_no_range(self):
+        declared = self._scored(models.MetricRange())
+        undeclared = self._scored(None)
+        self.assertEqual(declared['value_range'], 'unbounded')
+        self.assertEqual(undeclared['value_range'], 'none')
+
+    def test_direction_and_polarity_come_from_the_comparison(self):
+        row = self._rows(_comparison())['passed_check']
+        self.assertEqual(row['metric_direction'], 'improved')
+        self.assertEqual(row['higher_is_better'], 'true')
+
+    def test_an_ungated_run_has_no_direction_at_all(self):
+        # 'none' rather than 'neutral', for the same reason gate_win uses it:
+        # never computed is not the same as measured and did not move.
+        for row in self._rows().values():
+            self.assertEqual(row['metric_direction'], 'none')
+            self.assertEqual(row['higher_is_better'], 'unknown')
+
+    def test_a_metric_outside_the_comparison_has_no_direction(self):
+        rows = self._rows(_comparison())
+        outside = [
+            row
+            for metric, row in rows.items()
+            if metric and metric != 'passed_check'
+        ]
+        self.assertTrue(outside, 'fixture scores only the win metric')
+        for row in outside:
+            self.assertEqual(row['metric_direction'], 'none')
+
+    def test_a_named_but_unscored_metric_carries_no_range(self):
+        row = self._rows(_comparison())['never_scored']
+        self.assertEqual(row['metric_kind'], 'none')
+        self.assertEqual(row['value_range'], 'none')
+
+    def test_every_row_carries_all_five_columns(self):
+        # A store with no nullable columns rejects a row missing one, and
+        # the Kafka path has no client to hand that rejection back to.
+        columns = {
+            'value_range',
+            'value_minimum',
+            'value_maximum',
+            'metric_direction',
+            'higher_is_better',
+        }
+        for row in store.score_rows(_run(with_failure=True), _comparison()):
+            self.assertLessEqual(columns, set(row), row.get('metric'))
+
+
 class GraderLookupTests(unittest.TestCase):
     def test_maps_registry_types_to_categories(self):
         types, scales = store.grader_lookups(

{evalcore-2.6.0 → evalcore-2.6.1}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.6.0"
+version = "2.6.1"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },