evalcore 2.4.0__tar.gz → 2.4.2__tar.gz

{evalcore-2.4.0 → evalcore-2.4.2}/CHANGELOG.md

@@ -6,6 +6,70 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.4.2] - 2026-08-09
+
+Every `llm_as_judge` row now describes what it measures.
+
+Shipped as a patch. It changes what published rows contain, so the 1.0.0
+policy would call it a minor; it goes out as 2.4.2 as a deliberate exception,
+alongside 2.4.1's judge-pin fix that it completes. No schema change - the
+columns already exist and were empty.
+
+### Fixed
+- A judge's dimension row (`<grader>.<dimension>`) carried no judge
+  information at all: no `judges.*`, and `judge_scale` of 0. So `0.400` on a
+  row could not say who scored it, on what scale, or that the raw number was
+  2 out of 5 - the scale lived only on `<grader>.overall`, and recovering the
+  raw point meant joining back to it.
+
+  Each dimension row now carries the verdict for the dimension it measures:
+  `judges.name`, `judges.version`, `judges.rationale`, `judge_scale`, a
+  `judges.points` **scoped to that dimension**, and a `judges.score` that is
+  that judge's number for it.
+
+  `<grader>.overall` is unchanged and still carries the full points map.
+
+  This establishes one invariant across every judge row, dimension rows
+  included: `value == mean(judges.score)`. With one judge they are equal;
+  with a panel, `value` is the mean and `judges.score` shows the spread - so
+  per-dimension disagreement is readable from the dimension row instead of
+  by unpacking the map on `.overall`.
+
+**Upgrading:** `notEmpty(judges.name)` now matches every judge row rather
+than only `<grader>.overall`, so a six-case run goes from 12 judged rows to
+48. Anything aggregating over judges must filter to `.overall` or it
+multiplies by the dimension count. That is a change in results, not an error.
+
+## [2.4.1] - 2026-08-09
+
+The judge's identity is now in its provenance pin.
+
+Shipped as a patch. Both changes below alter a published value, so the 1.0.0
+policy would call this a minor at least; it goes out as 2.4.1 as a deliberate
+exception, because the old pin was answering a provenance question wrongly
+and the sooner it stops the fewer runs are affected.
+
+### Fixed
+- `Scorecard.judge_version` includes the judge's model:
+  `anthropic:claude-sonnet-4-6@v1` rather than `judge@v1`. It was
+  `key@judge_version`, so swapping the judge's model while leaving the
+  declared `judge_version` alone produced a byte-identical pin - and any
+  provenance check reading that field passed a comparison against a baseline
+  scored by a different model. The model is the thing most likely to change
+  and the thing a declared version is most likely to miss.
+
+### Changed
+- A single judge's key defaults to its provider (`anthropic`) instead of the
+  literal `judge`. A panel already defaulted to `key or provider`, so the
+  one-judge case was the odd one out, and `judge` named nothing the `grader`
+  column did not already say. It appears in the `judges.name` column.
+
+**Upgrading:** every suite with a judge re-baselines once, because
+`judge_version` is part of what identifies a comparable run - which is the
+intended behaviour, just paid all at once. Queries filtering
+`judges.name = 'judge'` need the provider instead. Set `key:` on a `judges:`
+entry to pin a name of your own.
+
 ## [2.4.0] - 2026-08-09
 
 A run without a baseline can now say whether it passed.
@@ -248,7 +312,9 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.2...HEAD
+[2.4.2]: https://github.com/scottpmiller/evalcore/compare/2.4.1...2.4.2
+[2.4.1]: https://github.com/scottpmiller/evalcore/compare/2.4.0...2.4.1
 [2.4.0]: https://github.com/scottpmiller/evalcore/compare/2.3.0...2.4.0
 [2.3.0]: https://github.com/scottpmiller/evalcore/compare/2.2.0...2.3.0
 [2.2.0]: https://github.com/scottpmiller/evalcore/compare/2.1.0...2.2.0

{evalcore-2.4.0 → evalcore-2.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.4.0
+Version: 2.4.2
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.4.0 → evalcore-2.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.4.0"
+version = "2.4.2"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.4.0 → evalcore-2.4.2}/src/evalcore/graders/judge.py

@@ -358,7 +358,11 @@ class RubricJudge:
         else:
             self.judges = [
                 {
-                    'key': 'judge',
+                    # The provider, not a literal 'judge'. A single judge is
+                    # a panel of one, so it names itself the same way a panel
+                    # member does; 'judge' carried no information and made
+                    # the one-judge case the odd one out.
+                    'key': 'anthropic',
                     'provider': 'anthropic',
                     'model': _env_expand(model),
                     'api_key_env': None,
@@ -394,17 +398,27 @@ class RubricJudge:
 
     @property
     def judge_version(self) -> str:
-        """Provenance pin: ``key@version`` per configured judge (a panel
-        joins them, comma-separated).
+        """Provenance pin: ``key:model@version`` per configured judge (a
+        panel joins them, comma-separated).
 
         The runner reads this onto ``Scorecard.judge_version`` so a judge
         model / prompt / scale change surfaces as a re-baseline event rather
         than hiding in each score's ``detail``. Uses the configured judges,
         not the mode-filtered active set, so the pin is stable across
         environments.
+
+        The model is in the pin because it is the thing most likely to
+        change and the thing a declared version is most likely to miss.
+        Without it, swapping the judge's model while leaving
+        ``judge_version`` alone produced a byte-identical pin - so a
+        comparison against a baseline scored by a different model passed
+        every provenance check that reads this field.
         """
         return ','.join(
-            f'{j["key"]}@{j["judge_version"]}' for j in self.judges
+            f'{j["key"]}:{j["model"]}@{j["judge_version"]}'
+            if j['model']
+            else f'{j["key"]}@{j["judge_version"]}'
+            for j in self.judges
         )
 
     def set_mode(self, mode: str) -> None:
@@ -565,6 +579,14 @@ class RubricJudge:
 
         # Panel mean per dimension (identical to the single judge's value
         # when there is only one).
+        #
+        # Each dimension row carries the verdict for the dimension it
+        # measures, not the whole one: points scoped to this key, and a score
+        # that is this judge's number for it. That makes `value ==
+        # mean(judges.score)` hold on every judge row, so per-dimension
+        # disagreement is readable without unpacking the map on `.overall`.
+        # Carrying the full map here would put two irrelevant numbers on a
+        # row and invite reading the wrong one.
         panel_dims: list[float] = []
         for key, _ in self.dimensions:
             per_judge = [
@@ -574,7 +596,19 @@ class RubricJudge:
             value = sum(present) / len(present) if present else None
             if value is not None:
                 panel_dims.append(value)
-            out.append(score(f'{self.name}.{key}', value))
+            dim_details = [
+                models.JudgeDetail(
+                    key=judge['key'],
+                    version=judge['judge_version'],
+                    rationale=rationales.get(judge['key']),
+                    points={key: raw.get(judge['key'], {}).get(key)},
+                    overall=self._normalize(
+                        raw.get(judge['key'], {}).get(key)
+                    ),
+                )
+                for judge in active
+            ]
+            out.append(score(f'{self.name}.{key}', value, judges=dim_details))
 
         overall = sum(panel_dims) / len(panel_dims) if panel_dims else None
         out.append(

{evalcore-2.4.0 → evalcore-2.4.2}/tests/test_judge.py

@@ -59,13 +59,62 @@ class RubricJudgeTests(unittest.TestCase):
         # raw 1..scale points are retained, not just the normalized mean
         self.assertEqual(jd.points, {'clarity': 5, 'specificity': 4})
         self.assertAlmostEqual(jd.overall, 0.9)
-        # only the overall score carries the breakdown; dimensions do not
-        dims = [
-            s
-            for s in _grade(grader, case, output)
-            if s.metric == 'llm_judge.clarity'
-        ]
-        self.assertEqual(dims[0].judges, [])
+        # the overall score carries the whole breakdown
+        self.assertEqual(jd.points, {'clarity': 5, 'specificity': 4})
+
+    def test_a_dimension_row_carries_its_own_verdict(self):
+        """Every judge row describes what that row measures.
+
+        The dimension row used to carry nothing, so `0.8` alone could not say
+        who scored it, on what scale, or that the raw number was 4.
+        """
+        grader = self._grader(
+            {
+                'Great subject': {
+                    'scores': {'clarity': 5, 'specificity': 4},
+                    'rationale': 'Clear and specific enough.',
+                }
+            }
+        )
+        scores = _grade(
+            grader,
+            models.Case(id='c1'),
+            models.Output(fields={'text': 'Great subject'}),
+        )
+        dim = next(s for s in scores if s.metric == 'llm_judge.specificity')
+        self.assertEqual(len(dim.judges), 1)
+        jd = dim.judges[0]
+        # Scoped: only the point this row is about.
+        self.assertEqual(jd.points, {'specificity': 4})
+        self.assertAlmostEqual(jd.overall, 0.8)
+        self.assertEqual(jd.rationale, 'Clear and specific enough.')
+
+    def test_value_is_the_mean_of_the_judge_scores(self):
+        """The invariant that holds on every judge row, dimensions included:
+        value == mean(judges.score)."""
+        grader = self._grader(
+            {'X': {'scores': {'clarity': 5, 'specificity': 2}}}
+        )
+        scores = _grade(
+            grader, models.Case(id='c'), models.Output(fields={'text': 'X'})
+        )
+        judged = [s for s in scores if s.judges and s.value is not None]
+        self.assertTrue(judged)
+        for s in judged:
+            mean = sum(j.overall for j in s.judges) / len(s.judges)
+            self.assertAlmostEqual(s.value, mean, msg=s.metric)
+
+    def test_the_overall_row_keeps_the_full_map(self):
+        grader = self._grader(
+            {'X': {'scores': {'clarity': 5, 'specificity': 4}}}
+        )
+        scores = _grade(
+            grader, models.Case(id='c'), models.Output(fields={'text': 'X'})
+        )
+        overall = next(s for s in scores if s.metric == 'llm_judge.overall')
+        self.assertEqual(
+            overall.judges[0].points, {'clarity': 5, 'specificity': 4}
+        )
 
     def test_different_content_gets_different_scores(self):
         grader = self._grader(
@@ -125,7 +174,8 @@ class RubricJudgeTests(unittest.TestCase):
             replay_path='f.yaml',
             judge_version='v3',
         )
-        self.assertEqual(single.judge_version, 'judge@v3')
+        # No model configured (replay-only), so the pin is key@version.
+        self.assertEqual(single.judge_version, 'anthropic@v3')
         panel = judge.RubricJudge(
             content_ref='output.text',
             dimensions=DIMENSIONS,
@@ -139,7 +189,37 @@ class RubricJudgeTests(unittest.TestCase):
                 },
             ],
         )
-        self.assertEqual(panel.judge_version, 'claude@v2,gpt@v5')
+        self.assertEqual(panel.judge_version, 'claude:m@v2,gpt:openai:g@v5')
+
+    def test_the_model_is_in_the_pin(self):
+        """Swapping the judge's model has to move the pin.
+
+        Without this, a model swap that left `judge_version` alone produced
+        a byte-identical pin, so a comparison against a baseline scored by a
+        different model passed every provenance check reading this field.
+        """
+
+        def _judge(model):
+            return judge.RubricJudge(
+                content_ref='output.text',
+                dimensions=DIMENSIONS,
+                model=model,
+                judge_version='v1',
+            )
+
+        before = _judge('claude-sonnet-4-6').judge_version
+        after = _judge('claude-opus-4-1').judge_version
+        self.assertEqual(before, 'anthropic:claude-sonnet-4-6@v1')
+        self.assertNotEqual(before, after)
+
+    def test_a_single_judge_names_itself_by_provider(self):
+        """Not the literal 'judge', which carried no information."""
+        single = judge.RubricJudge(
+            content_ref='output.text',
+            dimensions=DIMENSIONS,
+            replay_path='f.yaml',
+        )
+        self.assertEqual(single.judges[0]['key'], 'anthropic')
 
     def test_transient_client_error_retried_via_set_retry(self):
         class _RateLimit(Exception):

{evalcore-2.4.0 → evalcore-2.4.2}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.4.0"
+version = "2.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },