evalcore 2.4.2__tar.gz → 2.4.3__tar.gz

{evalcore-2.4.2 → evalcore-2.4.3}/CHANGELOG.md

@@ -6,6 +6,34 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [2.4.3] - 2026-08-11
+
+Live Anthropic judges work on current Claude models again.
+
+### Fixed
+- The Anthropic judge and pairwise clients no longer send `temperature=0`.
+  `temperature` (with `top_p`/`top_k`) was removed from the Claude request
+  surface at Opus 4.7, and sending it at all is a 400 there and on every model
+  after it - so a judge or a pairwise comparison pinned to `claude-opus-4-7`,
+  `claude-opus-4-8`, `claude-opus-5`, `claude-sonnet-5` or `claude-fable-5`
+  failed every call. The OpenAI clients still send it; that API still takes it.
+
+### Changed
+- Live judge `max_tokens` defaults are now 8192 (from 1024 on the rubric
+  judge, 512 on pairwise). Thinking is on by default from Opus 5 and Sonnet 5
+  onward and `max_tokens` bounds thinking plus reply together, so a
+  1024-token budget could be spent on reasoning before the forced tool call
+  landed - which surfaced as error scores rather than as an error. It is a
+  ceiling, not a spend: a model that does not think generates the same handful
+  of tokens it did before.
+
+**Upgrading:** a judge on a model that still accepts `temperature` (Sonnet
+4.6, Opus 4.6, the 4.5 line and older) now samples at the API default instead
+of 0, so its scores are no longer pinned run to run - expect more variance in
+a rubric dimension or a win-rate than before, and re-baseline if a gate sits
+close to its threshold. A judge on a thinking model also now bills thinking
+tokens on every call. Pass `max_tokens=` to a client to keep the old budget.
+
 ## [2.4.2] - 2026-08-09
 
 Every `llm_as_judge` row now describes what it measures.
@@ -312,7 +340,8 @@ by semantic versioning: a breaking change to either means a 2.0.
   rating + ranking with judge agreement, Markdown/HTML reporters, JSON +
   column-store outbox, and content-hash provenance.
 
-[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.2...HEAD
+[Unreleased]: https://github.com/scottpmiller/evalcore/compare/2.4.3...HEAD
+[2.4.3]: https://github.com/scottpmiller/evalcore/compare/2.4.2...2.4.3
 [2.4.2]: https://github.com/scottpmiller/evalcore/compare/2.4.1...2.4.2
 [2.4.1]: https://github.com/scottpmiller/evalcore/compare/2.4.0...2.4.1
 [2.4.0]: https://github.com/scottpmiller/evalcore/compare/2.3.0...2.4.0

{evalcore-2.4.2 → evalcore-2.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: evalcore
-Version: 2.4.2
+Version: 2.4.3
 Summary: A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs.
 Project-URL: Homepage, https://github.com/scottpmiller/evalcore
 Project-URL: Repository, https://github.com/scottpmiller/evalcore
@@ -327,7 +327,7 @@ degrade to `null`. `compare`'s guardrails and a `win_metric` with
 `generation_cost` or `tool_error_rate` alongside quality judges.
 
 The judge runs live (`AnthropicJudgeClient` forced tool call, or
-`OpenAIJudgeClient` `json_schema` - both temperature 0, needing the `judge`
+`OpenAIJudgeClient` `json_schema` at temperature 0 - needing the `judge`
 extra plus `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`) or offline
 (`ReplayJudgeClient`), chosen by the run mode like the adapter. Each
 dimension becomes a metric `<name>.<key>` plus a `<name>.overall` mean.

{evalcore-2.4.2 → evalcore-2.4.3}/README.md

@@ -295,7 +295,7 @@ degrade to `null`. `compare`'s guardrails and a `win_metric` with
 `generation_cost` or `tool_error_rate` alongside quality judges.
 
 The judge runs live (`AnthropicJudgeClient` forced tool call, or
-`OpenAIJudgeClient` `json_schema` - both temperature 0, needing the `judge`
+`OpenAIJudgeClient` `json_schema` at temperature 0 - needing the `judge`
 extra plus `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`) or offline
 (`ReplayJudgeClient`), chosen by the run mode like the adapter. Each
 dimension becomes a metric `<name>.<key>` plus a `<name>.overall` mean.

{evalcore-2.4.2 → evalcore-2.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "evalcore"
-version = "2.4.2"
+version = "2.4.3"
 description = "A generic, consumer-agnostic evaluation engine for prompt, model, and API outputs."
 readme = "README.md"
 requires-python = ">=3.11"

{evalcore-2.4.2 → evalcore-2.4.3}/src/evalcore/graders/judge.py

@@ -104,7 +104,7 @@ class AnthropicJudgeClient:
         self,
         model: str,
         api_key_env: str = 'ANTHROPIC_API_KEY',
-        max_tokens: int = 1024,
+        max_tokens: int = 8192,
         timeout: float = 30.0,
     ):
         self.model = model
@@ -156,7 +156,6 @@ class AnthropicJudgeClient:
             response = await client.messages.create(
                 model=self.model,
                 max_tokens=self.max_tokens,
-                temperature=0,
                 timeout=self.timeout,
                 system=system,
                 tools=[tool],
@@ -179,7 +178,7 @@ class OpenAIJudgeClient:
         self,
         model: str,
         api_key_env: str = 'OPENAI_API_KEY',
-        max_tokens: int = 1024,
+        max_tokens: int = 8192,
         timeout: float = 30.0,
     ):
         # Accept a 'provider:model' id (e.g. 'openai:gpt-4o'); SDK wants bare.
@@ -326,7 +325,7 @@ class RubricJudge:
         judge_version: str = 'v1',
         replay_path: str | None = None,
         client: JudgeClient | None = None,
-        max_tokens: int = 1024,
+        max_tokens: int = 8192,
     ):
         self.name = name
         self.content_ref = content_ref

{evalcore-2.4.2 → evalcore-2.4.3}/src/evalcore/pairwise.py

@@ -70,7 +70,7 @@ class AnthropicPairwiseClient:
         self,
         model: str,
         api_key_env: str = 'ANTHROPIC_API_KEY',
-        max_tokens: int = 512,
+        max_tokens: int = 8192,
         timeout: float = 30.0,
     ):
         self.model = model
@@ -88,7 +88,6 @@ class AnthropicPairwiseClient:
             response = await client.messages.create(
                 model=self.model,
                 max_tokens=self.max_tokens,
-                temperature=0,
                 timeout=self.timeout,
                 system=system,
                 tools=[tool],
@@ -111,7 +110,7 @@ class OpenAIPairwiseClient:
         self,
         model: str,
         api_key_env: str = 'OPENAI_API_KEY',
-        max_tokens: int = 512,
+        max_tokens: int = 8192,
         timeout: float = 30.0,
     ):
         self.model = model.split(':', 1)[1] if ':' in model else model

{evalcore-2.4.2 → evalcore-2.4.3}/uv.lock

@@ -188,7 +188,7 @@ wheels = [
 
 [[package]]
 name = "evalcore"
-version = "2.4.2"
+version = "2.4.3"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },