evalcore 2.4.0__tar.gz → 2.4.1__tar.gz

{evalcore-2.4.0 → evalcore-2.4.1}/CHANGELOG.md

@@ -6,6 +6,36 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.4.1] - 2026-08-09
+
+The judge's identity is now in its provenance pin.
+
+Shipped as a patch. Both changes below alter a published value, so the 1.0.0
+policy would call this a minor at least; it goes out as 2.4.1 as a deliberate
+exception, because the old pin was answering a provenance question wrongly
+and the sooner it stops the fewer runs are affected.
+
+### Fixed
+- `Scorecard.judge_version` includes the judge's model:
+  `anthropic:claude-sonnet-4-6@v1` rather than `judge@v1`. It was
+  `key@judge_version`, so swapping the judge's model while leaving the
+  declared `judge_version` alone produced a byte-identical pin - and any
+  provenance check reading that field passed a comparison against a baseline
+  scored by a different model. The model is the thing most likely to change
+  and the thing a declared version is most likely to miss.
+
+### Changed
+- A single judge's key defaults to its provider (`anthropic`) instead of the
+  literal `judge`. A panel already defaulted to `key or provider`, so the
+  one-judge case was the odd one out, and `judge` named nothing the `grader`
+  column did not already say. It appears in the `judges.name` column.
+
+**Upgrading:** every suite with a judge re-baselines once, because
+`judge_version` is part of what identifies a comparable run - which is the
+intended behaviour, just paid all at once. Queries filtering
+`judges.name = 'judge'` need the provider instead. Set `key:` on a `judges:`
+entry to pin a name of your own.
+
 ## [2.4.0] - 2026-08-09
 
 A run without a baseline can now say whether it passed.
@@ -248,7 +278,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.0...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.1...HEAD
+[2.4.1]: https://github.com/scottpmiller/evalcore/compare/2.4.0...2.4.1
 [2.4.0]: https://github.com/scottpmiller/evalcore/compare/2.3.0...2.4.0
 [2.3.0]: https://github.com/scottpmiller/evalcore/compare/2.2.0...2.3.0
 [2.2.0]: https://github.com/scottpmiller/evalcore/compare/2.1.0...2.2.0

{evalcore-2.4.0 → evalcore-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalcore
-Version: 2.4.0
+Version: 2.4.1
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore

{evalcore-2.4.0 → evalcore-2.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.4.0"
+version = "2.4.1"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.4.0 → evalcore-2.4.1}/src/evalcore/graders/judge.py

@@ -358,7 +358,11 @@ class RubricJudge:
         else:
             self.judges = [
                 {
-                    'key': 'judge',
+                    # The provider, not a literal 'judge'. A single judge is
+                    # a panel of one, so it names itself the same way a panel
+                    # member does; 'judge' carried no information and made
+                    # the one-judge case the odd one out.
+                    'key': 'anthropic',
                     'provider': 'anthropic',
                     'model': _env_expand(model),
                     'api_key_env': None,
@@ -394,17 +398,27 @@ class RubricJudge:
 
     @property
     def judge_version(self) -> str:
-        """Provenance pin: ``key@version`` per configured judge (a panel
-        joins them, comma-separated).
+        """Provenance pin: ``key:model@version`` per configured judge (a
+        panel joins them, comma-separated).
 
         The runner reads this onto ``Scorecard.judge_version`` so a judge
         model / prompt / scale change surfaces as a re-baseline event rather
         than hiding in each score's ``detail``. Uses the configured judges,
         not the mode-filtered active set, so the pin is stable across
         environments.
+
+        The model is in the pin because it is the thing most likely to
+        change and the thing a declared version is most likely to miss.
+        Without it, swapping the judge's model while leaving
+        ``judge_version`` alone produced a byte-identical pin - so a
+        comparison against a baseline scored by a different model passed
+        every provenance check that reads this field.
         """
         return ','.join(
-            f'{j["key"]}@{j["judge_version"]}' for j in self.judges
+            f'{j["key"]}:{j["model"]}@{j["judge_version"]}'
+            if j['model']
+            else f'{j["key"]}@{j["judge_version"]}'
+            for j in self.judges
         )
 
     def set_mode(self, mode: str) -> None:

{evalcore-2.4.0 → evalcore-2.4.1}/tests/test_judge.py

@@ -125,7 +125,8 @@ class RubricJudgeTests(unittest.TestCase):
             replay_path='f.yaml',
             judge_version='v3',
         )
-        self.assertEqual(single.judge_version, 'judge@v3')
+        # No model configured (replay-only), so the pin is key@version.
+        self.assertEqual(single.judge_version, 'anthropic@v3')
         panel = judge.RubricJudge(
             content_ref='output.text',
             dimensions=DIMENSIONS,
@@ -139,7 +140,37 @@ class RubricJudgeTests(unittest.TestCase):
                 },
             ],
         )
-        self.assertEqual(panel.judge_version, 'claude@v2,gpt@v5')
+        self.assertEqual(panel.judge_version, 'claude:m@v2,gpt:openai:g@v5')
+
+    def test_the_model_is_in_the_pin(self):
+        """Swapping the judge's model has to move the pin.
+
+        Without this, a model swap that left `judge_version` alone produced
+        a byte-identical pin, so a comparison against a baseline scored by a
+        different model passed every provenance check reading this field.
+        """
+
+        def _judge(model):
+            return judge.RubricJudge(
+                content_ref='output.text',
+                dimensions=DIMENSIONS,
+                model=model,
+                judge_version='v1',
+            )
+
+        before = _judge('claude-sonnet-4-6').judge_version
+        after = _judge('claude-opus-4-1').judge_version
+        self.assertEqual(before, 'anthropic:claude-sonnet-4-6@v1')
+        self.assertNotEqual(before, after)
+
+    def test_a_single_judge_names_itself_by_provider(self):
+        """Not the literal 'judge', which carried no information."""
+        single = judge.RubricJudge(
+            content_ref='output.text',
+            dimensions=DIMENSIONS,
+            replay_path='f.yaml',
+        )
+        self.assertEqual(single.judges[0]['key'], 'anthropic')
 
     def test_transient_client_error_retried_via_set_retry(self):
         class _RateLimit(Exception):

{evalcore-2.4.0 → evalcore-2.4.1}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.4.0"
+version = "2.4.1"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },