evalstone 0.2.1__tar.gz → 0.2.3__tar.gz

{evalstone-0.2.1 → evalstone-0.2.3}/CHANGELOG.md

@@ -2,6 +2,40 @@
 
 All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
 
+## 0.2.3 — 2026-10-14
+
+Promote pins the model the verdicts were routed through, found on the demo project's first
+real calibration.
+
+- `judge` and `comparison.pairwise` record `raw.judge_model`: the `<provider>/<model>` the
+  verdict was routed through (for example `openrouter/anthropic/claude-haiku-5.5`), next to
+  `raw.model`, the id the provider reported (`anthropic/claude-haiku-5.5`). Verdicts served
+  from the cache carry it too.
+- The center stamps only `raw.judge_model`, so the Judges page, calibrations and **Promote**
+  use the routed model. Before, it stamped `raw.model`, and Promote sent that
+  provider-reported id, which judge settings reject (`no provider named 'anthropic'`) or,
+  when a provider of that name existed, route elsewhere. Results pushed by an older library
+  are no longer stamped (a migration clears the old stamps): a version seen only from them
+  shows no model and cannot be promoted until a 0.2.3+ run pushes it.
+
+## 0.2.2 — 2026-10-14
+
+Judge replies cut off by the token cap, found by the first run against a real
+vendor (Claude Haiku 5.5 via OpenRouter on the rag-bot demo: 4 of 10 verdicts
+errored).
+
+- `judge` and `comparison.pairwise` default `max_tokens` is now 1024 (was 256).
+  The cut replies needed 286–349 tokens: a verdict plus a long one-sentence reason.
+  `max_tokens` is not part of the judge version, so cached verdicts stay valid.
+- A reply the provider stopped at the cap (`length`, `max_tokens`, `MAX_TOKENS`)
+  whose JSON is unclosed or invalid now errors with "reply truncated at
+  max_tokens=N; raise the check's max_tokens" followed by the original parse error and excerpt
+  instead of only "no JSON object in judge reply". A complete verdict at the cap is
+  accepted, and a complete but wrong one (score out of range) keeps its own error.
+  `ChatResponse.truncated` exposes the check.
+- `comparison.pairwise`: an unparseable judge reply now counts that call's cost
+  (and latency) in the errored score, as `judge` already did.
+
 ## 0.2.1 — 2026-10-14
 
 Attach fixes found by the docs audit.

{evalstone-0.2.1 → evalstone-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalstone
-Version: 0.2.1
+Version: 0.2.3
 Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
 Project-URL: Homepage, https://evalstone.com
 Project-URL: Documentation, https://evalstone.com/docs

{evalstone-0.2.1 → evalstone-0.2.3}/src/evalstone/__init__.py

@@ -8,7 +8,7 @@ import types
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"
 
 # Exports resolve on first access (PEP 562) so that importing the package — which
 # the pytest plugin does in every pytest session on the interpreter — costs

{evalstone-0.2.1 → evalstone-0.2.3}/src/evalstone/checks/comparison.py

@@ -12,7 +12,7 @@ from typing import Any
 
 from evalstone._json import sha256
 from evalstone.checks._base import FunctionCheck, judge_model_for
-from evalstone.checks.judge import _json_text, extract_json
+from evalstone.checks.judge import DEFAULT_MAX_TOKENS, _json_text, explain_unparsed, extract_json
 from evalstone.dataset import Sample
 from evalstone.output import Output
 from evalstone.providers import ChatMessage, ProviderError
@@ -46,6 +46,14 @@ def parse_winner(text: str) -> tuple[str, str]:
     return winner, str(data.get("reason", "")).strip()
 
 
+class _Unparsed(ValueError):
+    """An unparseable judge reply, with what the call cost."""
+
+    def __init__(self, message: str, cost_usd: float, latency_ms: float):
+        super().__init__(message)
+        self.cost_usd, self.latency_ms = cost_usd, latency_ms
+
+
 def pairwise(
     key: str,
     rubric: str,
@@ -53,7 +61,7 @@ def pairwise(
     baseline: Target | Mapping[str, Any],
     model: str | None = None,
     both_orders: bool = True,
-    max_tokens: int = 256,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
 ) -> FunctionCheck:
     """The candidate output beats the ``baseline`` output for the same sample.
 
@@ -78,7 +86,13 @@ def pairwise(
                 "max_tokens": max_tokens,
             },
         )
-        winner, reason = parse_winner(resp.text)
+        try:
+            winner, reason = parse_winner(resp.text)
+        except ValueError as exc:
+            # the failed call was still billed: carry its cost to the errored score
+            raise _Unparsed(
+                explain_unparsed(exc, resp, max_tokens), resp.cost_usd or 0.0, resp.latency_ms
+            ) from None
         if resp.cost_usd is None:
             unpriced.append(resp.model)
         return winner, reason, resp.cost_usd or 0.0, resp.latency_ms
@@ -101,7 +115,7 @@ def pairwise(
         cached = ctx.cache.get(cache_key) if ctx.cache is not None else None
         if cached is not None:
             # The judge verdict is cached; the baseline was still produced (its cost counts).
-            raw = {**(cached.raw or {}), "cached": True}
+            raw = {**(cached.raw or {}), "judge_model": resolved, "cached": True}
             return cached.model_copy(
                 update={"cost_usd": base.cost_usd, "latency_ms": base.latency_ms, "raw": raw}
             )
@@ -127,6 +141,12 @@ def pairwise(
                 reasons.append(reason)
         except ProviderError as exc:
             return Score.errored(str(exc), cost_usd=cost, latency_ms=latency)
+        except _Unparsed as exc:
+            return Score.errored(
+                f"pairwise: {exc}",
+                cost_usd=cost + exc.cost_usd,
+                latency_ms=latency + exc.latency_ms,
+            )
         except ValueError as exc:
             return Score.errored(f"pairwise: {exc}", cost_usd=cost, latency_ms=latency)
         value = sum(points) / len(points)
@@ -141,6 +161,7 @@ def pairwise(
                 "points": points,
                 "orders": len(points),
                 "judge_version": version,
+                "judge_model": resolved,
                 "unpriced": bool(unpriced),
             },
         )

{evalstone-0.2.1 → evalstone-0.2.3}/src/evalstone/checks/judge.py

@@ -16,11 +16,15 @@ from evalstone._json import dumps, sha256
 from evalstone.checks._base import FunctionCheck, judge_model_for
 from evalstone.dataset import Sample
 from evalstone.output import Output
-from evalstone.providers import ChatMessage, ProviderError
+from evalstone.providers import ChatMessage, ChatResponse, ProviderError
 from evalstone.score import CheckKind, Context, Score
 
 PROMPT_VERSION = 1
 
+# Room for a verdict plus a long reason: 256 cut real Haiku 5.5 replies mid-reason
+# (4 of 10 on the rag-bot demo). The cap only bounds a runaway; it is not in the version.
+DEFAULT_MAX_TOKENS = 1024
+
 SYSTEM_TEMPLATE = """You are a strict evaluator grading one answer produced by an AI system.
 
 RUBRIC:
@@ -67,6 +71,22 @@ def parse_verdict(text: str, scale: int) -> tuple[int, str]:
     return score, str(data.get("reason", "")).strip()
 
 
+def explain_unparsed(exc: ValueError, resp: ChatResponse, max_tokens: int) -> str:
+    """The parse error, prefixed with the cause when the cap cut the JSON itself.
+
+    Only an unclosed or invalid object is blamed on the cap: a complete object that is
+    wrong (score out of range, no winner) is the model's answer, not a truncation.
+    """
+    if resp.truncated:
+        try:
+            extract_json(resp.text)
+        except ValueError:
+            return (
+                f"reply truncated at max_tokens={max_tokens}; raise the check's max_tokens ({exc})"
+            )
+    return str(exc)
+
+
 class JudgeCheck(FunctionCheck):
     """A ``FunctionCheck`` whose function calls a model; exposes the judge version hash."""
 
@@ -99,7 +119,7 @@ def judge(
     pass_level: int | None = None,
     threshold: float | None = None,
     include_reference: bool = False,
-    max_tokens: int = 256,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
 ) -> JudgeCheck:
     """Grade an output against ``rubric`` with a model.
 
@@ -159,7 +179,15 @@ def judge(
                 reason=f"{n}/{scale}: {reason}" if reason else f"{n}/{scale}",
                 cost_usd=cost,
                 latency_ms=ms,
-                raw={**raw, "score": n, "scale": scale, "judge_version": judge_version},
+                # judge_model = `<provider>/<model>` as routed (what Promote pins); set here so
+                # a verdict served from an older cache entry carries it too
+                raw={
+                    **raw,
+                    "score": n,
+                    "scale": scale,
+                    "judge_version": judge_version,
+                    "judge_model": resolved,
+                },
             )
 
         # The cache holds the model's raw verdict (level + reason), keyed by the
@@ -190,13 +218,15 @@ def judge(
             n, reason = parse_verdict(resp.text, scale)
         except ValueError as exc:
             return Score.errored(
-                f"judge: {exc}", cost_usd=resp.cost_usd or 0.0, latency_ms=resp.latency_ms
+                f"judge: {explain_unparsed(exc, resp, max_tokens)}",
+                cost_usd=resp.cost_usd or 0.0,
+                latency_ms=resp.latency_ms,
             )
         score = to_score(
             n,
             reason,
             raw={
-                "model": resp.model,
+                "model": resp.model,  # what the provider reported (a bare id, maybe dated)
                 "tokens": [resp.input_tokens, resp.output_tokens],
                 "unpriced": resp.cost_usd is None,
                 "verdict_reason": reason,
@@ -217,4 +247,12 @@ def _verdict_reason(score: Score) -> str:
     return str((score.raw or {}).get("verdict_reason", ""))
 
 
-__all__ = ["PROMPT_VERSION", "JudgeCheck", "extract_json", "judge", "parse_verdict"]
+__all__ = [
+    "DEFAULT_MAX_TOKENS",
+    "PROMPT_VERSION",
+    "JudgeCheck",
+    "explain_unparsed",
+    "extract_json",
+    "judge",
+    "parse_verdict",
+]

{evalstone-0.2.1 → evalstone-0.2.3}/src/evalstone/providers/__init__.py

@@ -42,6 +42,11 @@ class ChatResponse(BaseModel):
     stop_reason: str | None = None
     raw: Any = None
 
+    @property
+    def truncated(self) -> bool:
+        """The provider stopped at the max_tokens cap (OpenAI-compatible, Anthropic, Gemini)."""
+        return (self.stop_reason or "").lower() in {"length", "max_tokens"}
+
 
 class ProviderError(Exception):
     """A provider call failed (HTTP error, refusal, malformed response)."""

{evalstone-0.2.1 → evalstone-0.2.3}/tests/test_judge.py

@@ -16,9 +16,9 @@ class FakeProvider:
 
     name = "openai"
 
-    def __init__(self, reply=None, fail=None, judge_reply=None):
+    def __init__(self, reply=None, fail=None, judge_reply=None, stop_reason=None):
         self.reply, self.fail, self.calls = reply, fail, []
-        self.judge_reply = judge_reply
+        self.judge_reply, self.stop_reason = judge_reply, stop_reason
 
     def complete(self, request):
         self.calls.append(request)
@@ -41,6 +41,7 @@ class FakeProvider:
             output_tokens=10,
             cost_usd=0.001,
             latency_ms=12.0,
+            stop_reason=self.stop_reason,
         )
 
 
@@ -114,6 +115,44 @@ def test_judge_errors_are_never_passes():
     assert no_model.error.startswith("no model given")
 
 
+# What a real Haiku 5.5 reply looked like when max_tokens=256 cut it mid-reason.
+CUT = '{"score": 4, "reason": "The answer directly answers the question with a concrete rule (30 da'
+
+
+@pytest.mark.parametrize("stop", ["length", "max_tokens", "MAX_TOKENS"])
+def test_a_reply_cut_at_the_token_cap_says_so_and_never_passes(stop):
+    chk = judge("h", "r", max_tokens=300)
+    score = chk(S, Output(value="x"), ctx_with(FakeProvider(judge_reply=CUT, stop_reason=stop)))
+    assert score.passed is False and score.cost_usd == 0.001
+    assert score.error.startswith(
+        "judge: reply truncated at max_tokens=300; raise the check's max_tokens "
+        "(no JSON object in judge reply: "
+    )
+    assert "concrete rule (30 da" in score.error  # the excerpt of what came back survives
+
+
+def test_truncation_only_explains_a_failed_parse():
+    # A complete verdict wins even if the provider then hit the cap.
+    full = ctx_with(
+        FakeProvider(judge_reply='{"score": 5, "reason": "ok"} and', stop_reason="length")
+    )
+    assert judge("h", "r")(S, Output(value="x"), full).passed is True
+    # A complete but wrong verdict at the cap is the model's answer, not a truncation.
+    wrong = ctx_with(
+        FakeProvider(judge_reply='{"score": 7, "reason": "x"} and', stop_reason="length")
+    )
+    assert judge("h", "r")(S, Output(value="x"), wrong).error == "judge: judge score 7 outside 1..5"
+    # A normal stop keeps the parse error as it was.
+    plain = ctx_with(FakeProvider(judge_reply=CUT, stop_reason="stop"))
+    assert "no JSON object" in judge("h", "r")(S, Output(value="x"), plain).error
+
+
+def test_judge_default_token_budget_fits_a_long_reason():
+    prov = FakeProvider()
+    judge("h", "r")(S, Output(value="30 days"), ctx_with(prov))
+    assert prov.calls[0].max_tokens == 1024
+
+
 def test_judge_cache_makes_the_second_identical_call_free():
     prov = FakeProvider()
     ctx = ctx_with(prov)
@@ -249,3 +288,20 @@ def test_judge_version_includes_the_resolved_default_model():
     b = chk(S, Output(value="Within 30 days."), ctx_with(prov, default="openai/judge-b"))
     assert a.raw["judge_version"] != b.raw["judge_version"]
     assert len(prov.calls) == 2  # different resolved model → different cache key
+
+
+def test_verdict_records_the_routed_model_fresh_and_cached():
+    """`judge_model` is what Promote pins: `<provider>/<model>` as routed, not `resp.model`."""
+    from evalstone.score import Score
+
+    chk = judge("h", "r")
+    ctx = ctx_with(FakeProvider())
+    fresh = chk(S, Output(value="30 days"), ctx)
+    assert fresh.raw["judge_model"] == "openai/judge-model"
+    assert chk(S, Output(value="30 days"), ctx).raw["judge_model"] == "openai/judge-model"
+    # a cache entry written by an older library (no judge_model) still gets it on the way out
+    key, old = next(iter(ctx.cache._data.items()))
+    ctx.cache._data[key] = Score.model_validate(
+        {**old.model_dump(), "raw": {k: v for k, v in old.raw.items() if k != "judge_model"}}
+    )
+    assert chk(S, Output(value="30 days"), ctx).raw["judge_model"] == "openai/judge-model"

{evalstone-0.2.1 → evalstone-0.2.3}/tests/test_rag_and_comparison.py

@@ -244,3 +244,43 @@ def test_config_baseline_mapping_form_and_target_form(tmp_path):
     c.providers.register("openai", JudgeFake(winner='{"winner": "A"}'))
     run = suite.run(target, ctx=c)
     assert run.summary.errors == 0 and run.summary.results == 2
+
+
+def test_pairwise_reply_cut_at_the_token_cap_says_so():
+    class Cut(JudgeFake):
+        def complete(self, request):
+            self.calls.append(request)
+            return ChatResponse(
+                text='{"winner": "A", "reason": "Answer A is more compl',
+                model=request.model,
+                cost_usd=0.001,
+                stop_reason="length",
+            )
+
+    fake = Cut()
+    score = pairwise("b", "r", baseline={"s": "x"})(S, Output(value="y"), ctx(fake))
+    assert score.passed is False
+    assert score.error.startswith("pairwise: reply truncated at max_tokens=1024")
+    assert fake.calls[0].max_tokens == 1024
+    assert score.cost_usd == pytest.approx(0.001)  # the failed call was billed: it counts
+
+
+def test_pairwise_unparseable_second_order_keeps_every_call_cost():
+    replies = iter(['{"winner": "A", "reason": "x"}', "no verdict here"])
+
+    class Half(JudgeFake):
+        def complete(self, request):
+            self.calls.append(request)
+            return ChatResponse(text=next(replies), model=request.model, cost_usd=0.001)
+
+    score = pairwise("b", "r", baseline={"s": "x"})(S, Output(value="y"), ctx(Half()))
+    assert score.passed is False and "no JSON object" in score.error
+    assert score.cost_usd == pytest.approx(0.002)
+
+
+def test_pairwise_records_the_routed_model_fresh_and_cached():
+    chk = pairwise("b", "r", baseline={"s": "x"})
+    c = ctx(JudgeFake())
+    assert chk(S, Output(value="y"), c).raw["judge_model"] == "openai/judge"
+    cached = chk(S, Output(value="y"), c)
+    assert cached.raw["cached"] is True and cached.raw["judge_model"] == "openai/judge"