evalstone 0.2.0__tar.gz → 0.2.1__tar.gz

{evalstone-0.2.0 → evalstone-0.2.1}/CHANGELOG.md

@@ -2,14 +2,30 @@
 
 All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
 
+## 0.2.1 — 2026-10-14
+
+Attach fixes found by the docs audit.
+
+- `comparison.pairwise` built its check version (and so its cache key) from
+  `--judge-model` or the check's own `model:`, ignoring the attached project's
+  per-check override — the call itself already honoured it. Both now resolve through
+  the same precedence (`--judge-model` > center override > `model:` > default), so a
+  verdict cached under the override is found again and a changed override invalidates
+  the cache.
+- Docstrings and this changelog no longer claim prompt targets are proxied when
+  attached; they never were (`providers:` + env keys serve the system under test).
+- Known limitation, documented: when attached, `--judge-model some-typo/model` is
+  not validated up front — the center only rejects the provider at the first judge
+  call, which surfaces as an errored score. Standalone runs still fail fast.
+
 ## 0.2.0 — 2026-10-14
 
 Attach a project to a center.
 
 - `center:` in `evalstone.yaml` (`url`, `project`, `api_key_env` — an `es_…` ingest key)
-  routes judge and prompt-target calls through the center's model proxy, so no provider
-  key lives on the machine running the suite; the project's judge settings pick the
-  model (per-check overrides, then the default), `--judge-model` still wins.
+  routes judge calls through the center's model proxy, so no judge key lives on the
+  machine running the suite; the project's judge settings pick the model (per-check
+  overrides, then the default), `--judge-model` still wins.
 - Judge verdicts are cached in the center (`CenterCache`); a center hiccup is a cache
   miss, never a failed run.
 - `human` checks of an attached project look labels up in the center (`labels: center`,

{evalstone-0.2.0 → evalstone-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalstone
-Version: 0.2.0
+Version: 0.2.1
 Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
 Project-URL: Homepage, https://evalstone.com
 Project-URL: Documentation, https://evalstone.com/docs

{evalstone-0.2.0 → evalstone-0.2.1}/src/evalstone/__init__.py

@@ -8,7 +8,7 @@ import types
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 # Exports resolve on first access (PEP 562) so that importing the package — which
 # the pytest plugin does in every pytest session on the interpreter — costs

{evalstone-0.2.0 → evalstone-0.2.1}/src/evalstone/center.py

@@ -1,11 +1,13 @@
 """Attach a project to an Evalstone center (``center:`` in evalstone.yaml).
 
-Attached, the library never holds a provider key: judge (and prompt-target) calls go
-through the center's model proxy, which resolves the model from the project's judge
-settings and dials the provider with the organization's key server-side. The judge
-cache lives in the center (one verdict serves every machine), ``human`` checks look
-labels up there, and every ``evalstone run`` pushes its report. ``providers:`` and
-``judges:`` in the YAML remain the standalone path.
+Attached, judge calls go through the center's model proxy, which resolves the model
+from the project's judge settings and dials the provider with the organization's key
+server-side, so no judge key lives on the machine running the suite. Prompt targets
+(the system under test) are NOT proxied: they keep using the local ``providers:`` and
+their env keys. The judge cache lives in the center (one verdict serves every
+machine), ``human`` checks look labels up there, and every ``evalstone run`` pushes
+its report. ``judges:`` in the YAML is ignored while attached; ``providers:`` still
+serves prompt targets.
 """
 
 from __future__ import annotations

{evalstone-0.2.0 → evalstone-0.2.1}/src/evalstone/checks/comparison.py

@@ -66,10 +66,12 @@ def pairwise(
     identity = {"v": 1, "rubric": rubric, "both_orders": both_orders}
     unpriced: list[str] = []  # models of this sample's calls that the price table lacks
 
-    def ask(ctx: Context, sample_input: str, a: str, b: str) -> tuple[str, str, float, float]:
+    def ask(
+        ctx: Context, chosen: str | None, sample_input: str, a: str, b: str
+    ) -> tuple[str, str, float, float]:
         user = USER.format(input=sample_input, a=a, b=b)
         resp = ctx.providers.complete(
-            judge_model_for(ctx, key, model),
+            chosen,
             {
                 "system": SYSTEM.format(rubric=rubric.strip()),
                 "messages": [ChatMessage(role="user", content=user)],
@@ -85,8 +87,9 @@ def pairwise(
         if ctx.providers is None:
             return Score.errored("pairwise: no providers configured")
         unpriced.clear()
+        chosen = judge_model_for(ctx, key, model)  # resolved once: the version AND the call
         try:
-            resolved = "/".join(ctx.providers.split(ctx.judge_model or model))
+            resolved = "/".join(ctx.providers.split(chosen))
         except ProviderError as exc:
             return Score.errored(str(exc))
         version = sha256({**identity, "model": resolved})[:16]
@@ -111,7 +114,7 @@ def pairwise(
             if both_orders:
                 orders.append(("base_first", base.text, output.text))
             for label, a, b in orders:
-                winner, reason, c, ms = ask(ctx, text, a, b)
+                winner, reason, c, ms = ask(ctx, chosen, text, a, b)
                 cost += c
                 latency += ms
                 candidate_is_a = label == "cand_first"

{evalstone-0.2.0 → evalstone-0.2.1}/src/evalstone/cli.py

@@ -145,6 +145,10 @@ def run(
     if judge_model is not None:
         try:
             registry = cfg.provider_registry()
+        except CenterError as exc:  # attached without a key: the center, not the flag
+            err.print(f"[red]{escape(str(exc))}[/red]")
+            raise typer.Exit(2) from None
+        try:
             registry.get(registry.split(judge_model)[0])
         except ProviderError as exc:
             err.print(f"[red]--judge-model: {escape(str(exc))}[/red]")

{evalstone-0.2.0 → evalstone-0.2.1}/src/evalstone/score.py

@@ -95,7 +95,8 @@ class ScoreCache(Protocol):
 
 
 class MemoryCache:
-    """An in-process ``ScoreCache``. The center-backed cache arrives with Phase 3."""
+    """An in-process ``ScoreCache`` (the standalone default; attached projects use
+    ``evalstone.center.CenterCache``)."""
 
     def __init__(self) -> None:
         self._data: dict[str, Score] = {}

{evalstone-0.2.0 → evalstone-0.2.1}/tests/test_center.py

@@ -312,6 +312,45 @@ def test_pairwise_honours_the_center_override(cfg):
     assert json.loads(calls.calls[0].request.content)["model"] == "other/big"
 
 
+@respx.mock
+def test_pairwise_cache_key_follows_the_center_override(cfg):
+    """The check version (and so the cache key) is built from the same resolved model as
+    the call: a verdict cached under the override is hit again, a changed override is a
+    different version and misses (regression: the key used ``ctx.judge_model or model``)."""
+    from evalstone.checks.comparison import pairwise
+    from evalstone.score import MemoryCache
+
+    calls = respx.post(f"{P}/model/complete").mock(
+        return_value=_verdict(text='{"winner": "A", "reason": "clearer"}')
+    )
+    client = CenterClient(cfg)
+    providers = CenterProviders(client, default_model="judge/small")
+    extra = {"center_client": client, "judge_models": {"better": "other/big"}}
+    cache = MemoryCache()
+    check = pairwise(
+        "better", "Which is clearer?", baseline={"s": "baseline answer"}, model="judge/small"
+    )
+    sample, output = Sample(id="s", input="q"), Output(value="candidate")
+
+    ctx = Context(providers=providers, extra=extra, cache=cache)
+    check.evaluate(sample, output, ctx)
+    filled = calls.call_count  # both orders were asked once
+    assert len(cache._data) == 1
+    key_before = next(iter(cache._data))
+    assert check.evaluate(sample, output, ctx).raw["cached"] is True
+    assert calls.call_count == filled  # served from the cache
+
+    ctx_other = Context(
+        providers=providers,
+        extra={**extra, "judge_models": {"better": "other/small"}},
+        cache=cache,
+    )
+    assert check.evaluate(sample, output, ctx_other).raw.get("cached") is not True
+    assert calls.call_count > filled  # a changed override is a different version → miss
+    assert json.loads(calls.calls[-1].request.content)["model"] == "other/small"
+    assert len(cache._data) == 2 and key_before in cache._data
+
+
 @respx.mock
 def test_prompt_targets_stay_local_when_attached(tmp_path, monkeypatch):
     from evalstone.providers import Providers

{evalstone-0.2.0 → evalstone-0.2.1}/tests/test_cli.py

@@ -143,3 +143,18 @@ def test_show_markdown(in_dir):
         "### Evalstone · `nightly` — ✅ gate passed"
     )
     assert "| `must_include` |" in res.output
+
+
+def test_missing_center_key_is_reported_as_the_center_not_the_flag(in_dir, monkeypatch):
+    """Attached without EVALSTONE_API_KEY, ``--judge-model`` must not blame the flag."""
+    monkeypatch.delenv("EVALSTONE_API_KEY", raising=False)
+    runner.invoke(app, ["init", "--project", "demo"])
+    cfg = in_dir / "evalstone.yaml"
+    cfg.write_text(
+        cfg.read_text()
+        + "center:\n  url: http://center.test\n  project: 11111111-2222-4333-8444-555555555555\n"
+    )
+    res = runner.invoke(app, ["run", "nightly", "--judge-model", "openai/gpt-x"])
+    assert res.exit_code == 2
+    assert "EVALSTONE_API_KEY" in res.output
+    assert "--judge-model" not in res.output

{evalstone-0.2.0 → evalstone-0.2.1}/tests/test_docs.py

@@ -51,3 +51,17 @@ def test_every_action_snippet_uses_the_same_ref():
             )
         )
     assert refs == {"action-v0"}, f"action refs in the docs: {sorted(refs)}"
+
+
+def test_site_changelog_is_generated_from_the_library_changelog():
+    """The site's Changelog page is `scripts/sync-changelog.mjs` output committed next to
+    the docs; a release that bumps CHANGELOG.md without re-running it is caught here."""
+    changelog = (SITE_DOCS.parents[2].parent / "python" / "evalstone" / "CHANGELOG.md").read_text(
+        encoding="utf-8"
+    )
+    page = (SITE_DOCS / "evalstone" / "changelog.mdx").read_text(encoding="utf-8")
+    body = changelog[changelog.index("\n## ") + 1 :].rstrip()
+    assert "GENERATED by scripts/sync-changelog.mjs" in page
+    assert page.rstrip().endswith(body), (
+        "run `npm run build` (or scripts/sync-changelog.mjs) in the site"
+    )