evalcore 2.4.1__tar.gz → 2.4.2__tar.gz

{evalcore-2.4.1 → evalcore-2.4.2}/CHANGELOG.md

@@ -6,6 +6,40 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.4.2] - 2026-08-09
+
+Every `llm_as_judge` row now describes what it measures.
+
+Shipped as a patch. It changes what published rows contain, so the 1.0.0
+policy would call it a minor; it goes out as 2.4.2 as a deliberate exception,
+alongside 2.4.1's judge-pin fix that it completes. No schema change - the
+columns already exist and were empty.
+
+### Fixed
+- A judge's dimension row (`<grader>.<dimension>`) carried no judge
+  information at all: no `judges.*`, and `judge_scale` of 0. So `0.400` on a
+  row could not say who scored it, on what scale, or that the raw number was
+  2 out of 5 - the scale lived only on `<grader>.overall`, and recovering the
+  raw point meant joining back to it.
+
+  Each dimension row now carries the verdict for the dimension it measures:
+  `judges.name`, `judges.version`, `judges.rationale`, `judge_scale`, a
+  `judges.points` **scoped to that dimension**, and a `judges.score` that is
+  that judge's number for it.
+
+  `<grader>.overall` is unchanged and still carries the full points map.
+
+  This establishes one invariant across every judge row, dimension rows
+  included: `value == mean(judges.score)`. With one judge they are equal;
+  with a panel, `value` is the mean and `judges.score` shows the spread - so
+  per-dimension disagreement is readable from the dimension row instead of
+  by unpacking the map on `.overall`.
+
+**Upgrading:** `notEmpty(judges.name)` now matches every judge row rather
+than only `<grader>.overall`, so a six-case run goes from 12 judged rows to
+48. Anything aggregating over judges must filter to `.overall` or it
+multiplies by the dimension count. That is a change in results, not an error.
+
 ## [2.4.1] - 2026-08-09
 
 The judge's identity is now in its provenance pin.
@@ -278,7 +312,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.1...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.2...HEAD
+[2.4.2]: https://github.com/scottpmiller/evalcore/compare/2.4.1...2.4.2
 [2.4.1]: https://github.com/scottpmiller/evalcore/compare/2.4.0...2.4.1
 [2.4.0]: https://github.com/scottpmiller/evalcore/compare/2.3.0...2.4.0
 [2.3.0]: https://github.com/scottpmiller/evalcore/compare/2.2.0...2.3.0

{evalcore-2.4.1 → evalcore-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.4.1
+Version: 2.4.2
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.4.1 → evalcore-2.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.4.1"
+version = "2.4.2"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.4.1 → evalcore-2.4.2}/src/evalcore/graders/judge.py

@@ -579,6 +579,14 @@ class RubricJudge:
 
         # Panel mean per dimension (identical to the single judge's value
         # when there is only one).
+        #
+        # Each dimension row carries the verdict for the dimension it
+        # measures, not the whole one: points scoped to this key, and a score
+        # that is this judge's number for it. That makes `value ==
+        # mean(judges.score)` hold on every judge row, so per-dimension
+        # disagreement is readable without unpacking the map on `.overall`.
+        # Carrying the full map here would put two irrelevant numbers on a
+        # row and invite reading the wrong one.
         panel_dims: list[float] = []
         for key, _ in self.dimensions:
             per_judge = [
@@ -588,7 +596,19 @@ class RubricJudge:
             value = sum(present) / len(present) if present else None
             if value is not None:
                 panel_dims.append(value)
-            out.append(score(f'{self.name}.{key}', value))
+            dim_details = [
+                models.JudgeDetail(
+                    key=judge['key'],
+                    version=judge['judge_version'],
+                    rationale=rationales.get(judge['key']),
+                    points={key: raw.get(judge['key'], {}).get(key)},
+                    overall=self._normalize(
+                        raw.get(judge['key'], {}).get(key)
+                    ),
+                )
+                for judge in active
+            ]
+            out.append(score(f'{self.name}.{key}', value, judges=dim_details))
 
         overall = sum(panel_dims) / len(panel_dims) if panel_dims else None
         out.append(

{evalcore-2.4.1 → evalcore-2.4.2}/tests/test_judge.py

@@ -59,13 +59,62 @@ class RubricJudgeTests(unittest.TestCase):
         # raw 1..scale points are retained, not just the normalized mean
         self.assertEqual(jd.points, {'clarity': 5, 'specificity': 4})
         self.assertAlmostEqual(jd.overall, 0.9)
-        # only the overall score carries the breakdown; dimensions do not
-        dims = [
-            s
-            for s in _grade(grader, case, output)
-            if s.metric == 'llm_judge.clarity'
-        ]
-        self.assertEqual(dims[0].judges, [])
+        # the overall score carries the whole breakdown
+        self.assertEqual(jd.points, {'clarity': 5, 'specificity': 4})
+
+    def test_a_dimension_row_carries_its_own_verdict(self):
+        """Every judge row describes what that row measures.
+
+        The dimension row used to carry nothing, so `0.8` alone could not say
+        who scored it, on what scale, or that the raw number was 4.
+        """
+        grader = self._grader(
+            {
+                'Great subject': {
+                    'scores': {'clarity': 5, 'specificity': 4},
+                    'rationale': 'Clear and specific enough.',
+                }
+            }
+        )
+        scores = _grade(
+            grader,
+            models.Case(id='c1'),
+            models.Output(fields={'text': 'Great subject'}),
+        )
+        dim = next(s for s in scores if s.metric == 'llm_judge.specificity')
+        self.assertEqual(len(dim.judges), 1)
+        jd = dim.judges[0]
+        # Scoped: only the point this row is about.
+        self.assertEqual(jd.points, {'specificity': 4})
+        self.assertAlmostEqual(jd.overall, 0.8)
+        self.assertEqual(jd.rationale, 'Clear and specific enough.')
+
+    def test_value_is_the_mean_of_the_judge_scores(self):
+        """The invariant that holds on every judge row, dimensions included:
+        value == mean(judges.score)."""
+        grader = self._grader(
+            {'X': {'scores': {'clarity': 5, 'specificity': 2}}}
+        )
+        scores = _grade(
+            grader, models.Case(id='c'), models.Output(fields={'text': 'X'})
+        )
+        judged = [s for s in scores if s.judges and s.value is not None]
+        self.assertTrue(judged)
+        for s in judged:
+            mean = sum(j.overall for j in s.judges) / len(s.judges)
+            self.assertAlmostEqual(s.value, mean, msg=s.metric)
+
+    def test_the_overall_row_keeps_the_full_map(self):
+        grader = self._grader(
+            {'X': {'scores': {'clarity': 5, 'specificity': 4}}}
+        )
+        scores = _grade(
+            grader, models.Case(id='c'), models.Output(fields={'text': 'X'})
+        )
+        overall = next(s for s in scores if s.metric == 'llm_judge.overall')
+        self.assertEqual(
+            overall.judges[0].points, {'clarity': 5, 'specificity': 4}
+        )
 
     def test_different_content_gets_different_scores(self):
         grader = self._grader(

{evalcore-2.4.1 → evalcore-2.4.2}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.4.1"
+version = "2.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },