evalstone 0.2.2__tar.gz → 0.2.3__tar.gz

{evalstone-0.2.2 → evalstone-0.2.3}/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
 
+## 0.2.3 — 2026-10-14
+
+Promote pins the model the verdicts were routed through, found on the demo project's first
+real calibration.
+
+- `judge` and `comparison.pairwise` record `raw.judge_model`: the `<provider>/<model>` the
+  verdict was routed through (for example `openrouter/anthropic/claude-haiku-5.5`), next to
+  `raw.model`, the id the provider reported (`anthropic/claude-haiku-5.5`). Verdicts served
+  from the cache carry it too.
+- The center stamps only `raw.judge_model`, so the Judges page, calibrations and **Promote**
+  use the routed model. Before, it stamped `raw.model`, and Promote sent that
+  provider-reported id, which judge settings reject (`no provider named 'anthropic'`) or,
+  when a provider of that name existed, route elsewhere. Results pushed by an older library
+  are no longer stamped (a migration clears the old stamps): a version seen only from them
+  shows no model and cannot be promoted until a 0.2.3+ run pushes it.
+
 ## 0.2.2 — 2026-10-14
 
 Judge replies cut off by the token cap, found by the first run against a real

{evalstone-0.2.2 → evalstone-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalstone
-Version: 0.2.2
+Version: 0.2.3
 Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
 Project-URL: Homepage, https://evalstone.com
 Project-URL: Documentation, https://evalstone.com/docs

{evalstone-0.2.2 → evalstone-0.2.3}/src/evalstone/__init__.py

@@ -8,7 +8,7 @@ import types
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 # Exports resolve on first access (PEP 562) so that importing the package — which
 # the pytest plugin does in every pytest session on the interpreter — costs

{evalstone-0.2.2 → evalstone-0.2.3}/src/evalstone/checks/comparison.py

@@ -115,7 +115,7 @@ def pairwise(
         cached = ctx.cache.get(cache_key) if ctx.cache is not None else None
         if cached is not None:
             # The judge verdict is cached; the baseline was still produced (its cost counts).
-            raw = {**(cached.raw or {}), "cached": True}
+            raw = {**(cached.raw or {}), "judge_model": resolved, "cached": True}
             return cached.model_copy(
                 update={"cost_usd": base.cost_usd, "latency_ms": base.latency_ms, "raw": raw}
             )
@@ -161,6 +161,7 @@ def pairwise(
                 "points": points,
                 "orders": len(points),
                 "judge_version": version,
+                "judge_model": resolved,
                 "unpriced": bool(unpriced),
             },
         )

{evalstone-0.2.2 → evalstone-0.2.3}/src/evalstone/checks/judge.py

@@ -179,7 +179,15 @@ def judge(
                 reason=f"{n}/{scale}: {reason}" if reason else f"{n}/{scale}",
                 cost_usd=cost,
                 latency_ms=ms,
-                raw={**raw, "score": n, "scale": scale, "judge_version": judge_version},
+                # judge_model = `<provider>/<model>` as routed (what Promote pins); set here so
+                # a verdict served from an older cache entry carries it too
+                raw={
+                    **raw,
+                    "score": n,
+                    "scale": scale,
+                    "judge_version": judge_version,
+                    "judge_model": resolved,
+                },
             )
 
         # The cache holds the model's raw verdict (level + reason), keyed by the
@@ -218,7 +226,7 @@ def judge(
             n,
             reason,
             raw={
-                "model": resp.model,
+                "model": resp.model,  # what the provider reported (a bare id, maybe dated)
                 "tokens": [resp.input_tokens, resp.output_tokens],
                 "unpriced": resp.cost_usd is None,
                 "verdict_reason": reason,

{evalstone-0.2.2 → evalstone-0.2.3}/tests/test_judge.py

@@ -288,3 +288,20 @@ def test_judge_version_includes_the_resolved_default_model():
     b = chk(S, Output(value="Within 30 days."), ctx_with(prov, default="openai/judge-b"))
     assert a.raw["judge_version"] != b.raw["judge_version"]
     assert len(prov.calls) == 2  # different resolved model → different cache key
+
+
+def test_verdict_records_the_routed_model_fresh_and_cached():
+    """`judge_model` is what Promote pins: `<provider>/<model>` as routed, not `resp.model`."""
+    from evalstone.score import Score
+
+    chk = judge("h", "r")
+    ctx = ctx_with(FakeProvider())
+    fresh = chk(S, Output(value="30 days"), ctx)
+    assert fresh.raw["judge_model"] == "openai/judge-model"
+    assert chk(S, Output(value="30 days"), ctx).raw["judge_model"] == "openai/judge-model"
+    # a cache entry written by an older library (no judge_model) still gets it on the way out
+    key, old = next(iter(ctx.cache._data.items()))
+    ctx.cache._data[key] = Score.model_validate(
+        {**old.model_dump(), "raw": {k: v for k, v in old.raw.items() if k != "judge_model"}}
+    )
+    assert chk(S, Output(value="30 days"), ctx).raw["judge_model"] == "openai/judge-model"

{evalstone-0.2.2 → evalstone-0.2.3}/tests/test_rag_and_comparison.py

@@ -276,3 +276,11 @@ def test_pairwise_unparseable_second_order_keeps_every_call_cost():
     score = pairwise("b", "r", baseline={"s": "x"})(S, Output(value="y"), ctx(Half()))
     assert score.passed is False and "no JSON object" in score.error
     assert score.cost_usd == pytest.approx(0.002)
+
+
+def test_pairwise_records_the_routed_model_fresh_and_cached():
+    chk = pairwise("b", "r", baseline={"s": "x"})
+    c = ctx(JudgeFake())
+    assert chk(S, Output(value="y"), c).raw["judge_model"] == "openai/judge"
+    cached = chk(S, Output(value="y"), c)
+    assert cached.raw["cached"] is True and cached.raw["judge_model"] == "openai/judge"