evalstone 0.2.1__tar.gz → 0.2.2__tar.gz

{evalstone-0.2.1 → evalstone-0.2.2}/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
 
+## 0.2.2 — 2026-10-14
+
+Judge replies cut off by the token cap, found by the first run against a real
+vendor (Claude Haiku 5.5 via OpenRouter on the rag-bot demo: 4 of 10 verdicts
+errored).
+
+- `judge` and `comparison.pairwise` default `max_tokens` is now 1024 (was 256).
+  The cut replies needed 286–349 tokens: a verdict plus a long one-sentence reason.
+  `max_tokens` is not part of the judge version, so cached verdicts stay valid.
+- A reply the provider stopped at the cap (`length`, `max_tokens`, `MAX_TOKENS`)
+  whose JSON is unclosed or invalid now errors with "reply truncated at
+  max_tokens=N; raise the check's max_tokens" followed by the original parse error and excerpt
+  instead of only "no JSON object in judge reply". A complete verdict at the cap is
+  accepted, and a complete but wrong one (score out of range) keeps its own error.
+  `ChatResponse.truncated` exposes the check.
+- `comparison.pairwise`: an unparseable judge reply now counts that call's cost
+  (and latency) in the errored score, as `judge` already did.
+
 ## 0.2.1 — 2026-10-14
 
 Attach fixes found by the docs audit.

{evalstone-0.2.1 → evalstone-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalstone
-Version: 0.2.1
+Version: 0.2.2
 Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
 Project-URL: Homepage, https://evalstone.com
 Project-URL: Documentation, https://evalstone.com/docs

{evalstone-0.2.1 → evalstone-0.2.2}/src/evalstone/__init__.py

@@ -8,7 +8,7 @@ import types
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-__version__ = "0.2.1"
+__version__ = "0.2.2"
 
 # Exports resolve on first access (PEP 562) so that importing the package — which
 # the pytest plugin does in every pytest session on the interpreter — costs

{evalstone-0.2.1 → evalstone-0.2.2}/src/evalstone/checks/comparison.py

@@ -12,7 +12,7 @@ from typing import Any
 
 from evalstone._json import sha256
 from evalstone.checks._base import FunctionCheck, judge_model_for
-from evalstone.checks.judge import _json_text, extract_json
+from evalstone.checks.judge import DEFAULT_MAX_TOKENS, _json_text, explain_unparsed, extract_json
 from evalstone.dataset import Sample
 from evalstone.output import Output
 from evalstone.providers import ChatMessage, ProviderError
@@ -46,6 +46,14 @@ def parse_winner(text: str) -> tuple[str, str]:
     return winner, str(data.get("reason", "")).strip()
 
 
+class _Unparsed(ValueError):
+    """An unparseable judge reply, with what the call cost."""
+
+    def __init__(self, message: str, cost_usd: float, latency_ms: float):
+        super().__init__(message)
+        self.cost_usd, self.latency_ms = cost_usd, latency_ms
+
+
 def pairwise(
     key: str,
     rubric: str,
@@ -53,7 +61,7 @@ def pairwise(
     baseline: Target | Mapping[str, Any],
     model: str | None = None,
     both_orders: bool = True,
-    max_tokens: int = 256,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
 ) -> FunctionCheck:
     """The candidate output beats the ``baseline`` output for the same sample.
 
@@ -78,7 +86,13 @@ def pairwise(
                 "max_tokens": max_tokens,
             },
         )
-        winner, reason = parse_winner(resp.text)
+        try:
+            winner, reason = parse_winner(resp.text)
+        except ValueError as exc:
+            # the failed call was still billed: carry its cost to the errored score
+            raise _Unparsed(
+                explain_unparsed(exc, resp, max_tokens), resp.cost_usd or 0.0, resp.latency_ms
+            ) from None
         if resp.cost_usd is None:
             unpriced.append(resp.model)
         return winner, reason, resp.cost_usd or 0.0, resp.latency_ms
@@ -127,6 +141,12 @@ def pairwise(
                 reasons.append(reason)
         except ProviderError as exc:
             return Score.errored(str(exc), cost_usd=cost, latency_ms=latency)
+        except _Unparsed as exc:
+            return Score.errored(
+                f"pairwise: {exc}",
+                cost_usd=cost + exc.cost_usd,
+                latency_ms=latency + exc.latency_ms,
+            )
         except ValueError as exc:
             return Score.errored(f"pairwise: {exc}", cost_usd=cost, latency_ms=latency)
         value = sum(points) / len(points)

{evalstone-0.2.1 → evalstone-0.2.2}/src/evalstone/checks/judge.py

@@ -16,11 +16,15 @@ from evalstone._json import dumps, sha256
 from evalstone.checks._base import FunctionCheck, judge_model_for
 from evalstone.dataset import Sample
 from evalstone.output import Output
-from evalstone.providers import ChatMessage, ProviderError
+from evalstone.providers import ChatMessage, ChatResponse, ProviderError
 from evalstone.score import CheckKind, Context, Score
 
 PROMPT_VERSION = 1
 
+# Room for a verdict plus a long reason: 256 cut real Haiku 5.5 replies mid-reason
+# (4 of 10 on the rag-bot demo). The cap only bounds a runaway; it is not in the version.
+DEFAULT_MAX_TOKENS = 1024
+
 SYSTEM_TEMPLATE = """You are a strict evaluator grading one answer produced by an AI system.
 
 RUBRIC:
@@ -67,6 +71,22 @@ def parse_verdict(text: str, scale: int) -> tuple[int, str]:
     return score, str(data.get("reason", "")).strip()
 
 
+def explain_unparsed(exc: ValueError, resp: ChatResponse, max_tokens: int) -> str:
+    """The parse error, prefixed with the cause when the cap cut the JSON itself.
+
+    Only an unclosed or invalid object is blamed on the cap: a complete object that is
+    wrong (score out of range, no winner) is the model's answer, not a truncation.
+    """
+    if resp.truncated:
+        try:
+            extract_json(resp.text)
+        except ValueError:
+            return (
+                f"reply truncated at max_tokens={max_tokens}; raise the check's max_tokens ({exc})"
+            )
+    return str(exc)
+
+
 class JudgeCheck(FunctionCheck):
     """A ``FunctionCheck`` whose function calls a model; exposes the judge version hash."""
 
@@ -99,7 +119,7 @@ def judge(
     pass_level: int | None = None,
     threshold: float | None = None,
     include_reference: bool = False,
-    max_tokens: int = 256,
+    max_tokens: int = DEFAULT_MAX_TOKENS,
 ) -> JudgeCheck:
     """Grade an output against ``rubric`` with a model.
 
@@ -190,7 +210,9 @@ def judge(
             n, reason = parse_verdict(resp.text, scale)
         except ValueError as exc:
             return Score.errored(
-                f"judge: {exc}", cost_usd=resp.cost_usd or 0.0, latency_ms=resp.latency_ms
+                f"judge: {explain_unparsed(exc, resp, max_tokens)}",
+                cost_usd=resp.cost_usd or 0.0,
+                latency_ms=resp.latency_ms,
             )
         score = to_score(
             n,
@@ -217,4 +239,12 @@ def _verdict_reason(score: Score) -> str:
     return str((score.raw or {}).get("verdict_reason", ""))
 
 
-__all__ = ["PROMPT_VERSION", "JudgeCheck", "extract_json", "judge", "parse_verdict"]
+__all__ = [
+    "DEFAULT_MAX_TOKENS",
+    "PROMPT_VERSION",
+    "JudgeCheck",
+    "explain_unparsed",
+    "extract_json",
+    "judge",
+    "parse_verdict",
+]

{evalstone-0.2.1 → evalstone-0.2.2}/src/evalstone/providers/__init__.py

@@ -42,6 +42,11 @@ class ChatResponse(BaseModel):
     stop_reason: str | None = None
     raw: Any = None
 
+    @property
+    def truncated(self) -> bool:
+        """The provider stopped at the max_tokens cap (OpenAI-compatible, Anthropic, Gemini)."""
+        return (self.stop_reason or "").lower() in {"length", "max_tokens"}
+
 
 class ProviderError(Exception):
     """A provider call failed (HTTP error, refusal, malformed response)."""

{evalstone-0.2.1 → evalstone-0.2.2}/tests/test_judge.py

@@ -16,9 +16,9 @@ class FakeProvider:
 
     name = "openai"
 
-    def __init__(self, reply=None, fail=None, judge_reply=None):
+    def __init__(self, reply=None, fail=None, judge_reply=None, stop_reason=None):
         self.reply, self.fail, self.calls = reply, fail, []
-        self.judge_reply = judge_reply
+        self.judge_reply, self.stop_reason = judge_reply, stop_reason
 
     def complete(self, request):
         self.calls.append(request)
@@ -41,6 +41,7 @@ class FakeProvider:
             output_tokens=10,
             cost_usd=0.001,
             latency_ms=12.0,
+            stop_reason=self.stop_reason,
         )
 
 
@@ -114,6 +115,44 @@ def test_judge_errors_are_never_passes():
     assert no_model.error.startswith("no model given")
 
 
+# What a real Haiku 5.5 reply looked like when max_tokens=256 cut it mid-reason.
+CUT = '{"score": 4, "reason": "The answer directly answers the question with a concrete rule (30 da'
+
+
+@pytest.mark.parametrize("stop", ["length", "max_tokens", "MAX_TOKENS"])
+def test_a_reply_cut_at_the_token_cap_says_so_and_never_passes(stop):
+    chk = judge("h", "r", max_tokens=300)
+    score = chk(S, Output(value="x"), ctx_with(FakeProvider(judge_reply=CUT, stop_reason=stop)))
+    assert score.passed is False and score.cost_usd == 0.001
+    assert score.error.startswith(
+        "judge: reply truncated at max_tokens=300; raise the check's max_tokens "
+        "(no JSON object in judge reply: "
+    )
+    assert "concrete rule (30 da" in score.error  # the excerpt of what came back survives
+
+
+def test_truncation_only_explains_a_failed_parse():
+    # A complete verdict wins even if the provider then hit the cap.
+    full = ctx_with(
+        FakeProvider(judge_reply='{"score": 5, "reason": "ok"} and', stop_reason="length")
+    )
+    assert judge("h", "r")(S, Output(value="x"), full).passed is True
+    # A complete but wrong verdict at the cap is the model's answer, not a truncation.
+    wrong = ctx_with(
+        FakeProvider(judge_reply='{"score": 7, "reason": "x"} and', stop_reason="length")
+    )
+    assert judge("h", "r")(S, Output(value="x"), wrong).error == "judge: judge score 7 outside 1..5"
+    # A normal stop keeps the parse error as it was.
+    plain = ctx_with(FakeProvider(judge_reply=CUT, stop_reason="stop"))
+    assert "no JSON object" in judge("h", "r")(S, Output(value="x"), plain).error
+
+
+def test_judge_default_token_budget_fits_a_long_reason():
+    prov = FakeProvider()
+    judge("h", "r")(S, Output(value="30 days"), ctx_with(prov))
+    assert prov.calls[0].max_tokens == 1024
+
+
 def test_judge_cache_makes_the_second_identical_call_free():
     prov = FakeProvider()
     ctx = ctx_with(prov)

{evalstone-0.2.1 → evalstone-0.2.2}/tests/test_rag_and_comparison.py

@@ -244,3 +244,35 @@ def test_config_baseline_mapping_form_and_target_form(tmp_path):
     c.providers.register("openai", JudgeFake(winner='{"winner": "A"}'))
     run = suite.run(target, ctx=c)
     assert run.summary.errors == 0 and run.summary.results == 2
+
+
+def test_pairwise_reply_cut_at_the_token_cap_says_so():
+    class Cut(JudgeFake):
+        def complete(self, request):
+            self.calls.append(request)
+            return ChatResponse(
+                text='{"winner": "A", "reason": "Answer A is more compl',
+                model=request.model,
+                cost_usd=0.001,
+                stop_reason="length",
+            )
+
+    fake = Cut()
+    score = pairwise("b", "r", baseline={"s": "x"})(S, Output(value="y"), ctx(fake))
+    assert score.passed is False
+    assert score.error.startswith("pairwise: reply truncated at max_tokens=1024")
+    assert fake.calls[0].max_tokens == 1024
+    assert score.cost_usd == pytest.approx(0.001)  # the failed call was billed: it counts
+
+
+def test_pairwise_unparseable_second_order_keeps_every_call_cost():
+    replies = iter(['{"winner": "A", "reason": "x"}', "no verdict here"])
+
+    class Half(JudgeFake):
+        def complete(self, request):
+            self.calls.append(request)
+            return ChatResponse(text=next(replies), model=request.model, cost_usd=0.001)
+
+    score = pairwise("b", "r", baseline={"s": "x"})(S, Output(value="y"), ctx(Half()))
+    assert score.passed is False and "no JSON object" in score.error
+    assert score.cost_usd == pytest.approx(0.002)