evalstone 0.1.1__tar.gz → 0.2.0__tar.gz

{evalstone-0.1.1 → evalstone-0.2.0}/.gitignore

@@ -42,3 +42,6 @@ npm-debug.log*
 
 # ── Evalstone run reports (written by `evalstone run`) ──────────────────
 **/evalstone-report.json
+
+# ── Celery beat state (written by `just beat`) ───────────────────────────
+**/celerybeat-schedule*

{evalstone-0.1.1 → evalstone-0.2.0}/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 All notable changes to `python/evalstone`. Semantic versioning; tags `lib-vX.Y.Z`.
 
+## 0.2.0 — 2026-10-14
+
+Attach a project to a center.
+
+- `center:` in `evalstone.yaml` (`url`, `project`, `api_key_env` — an `es_…` ingest key)
+  routes judge and prompt-target calls through the center's model proxy, so no provider
+  key lives on the machine running the suite; the project's judge settings pick the
+  model (per-check overrides, then the default), `--judge-model` still wins.
+- Judge verdicts are cached in the center (`CenterCache`); a center hiccup is a cache
+  miss, never a failed run.
+- `human` checks of an attached project look labels up in the center (`labels: center`,
+  the default when attached; a file path still works).
+- `evalstone run` pushes its report to the center (`push_runs: false` to opt out); a
+  failed push warns and leaves the exit code alone.
+- Prompt targets (the system under test) always use the local `providers:` + env keys,
+  even when attached — the center proxy is for judges only. The MCP server honours the
+  attach too.
+
 ## 0.1.1 — 2026-10-13
 
 Judge switching and cost honesty.

{evalstone-0.1.1 → evalstone-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: evalstone
-Version: 0.1.1
+Version: 0.2.0
 Summary: The plug-and-play evals layer for LLM-based projects: datasets, checks, suites, runs and gates.
 Project-URL: Homepage, https://evalstone.com
 Project-URL: Documentation, https://evalstone.com/docs

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/__init__.py

@@ -8,7 +8,7 @@ import types
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-__version__ = "0.1.1"
+__version__ = "0.2.0"
 
 # Exports resolve on first access (PEP 562) so that importing the package — which
 # the pytest plugin does in every pytest session on the interpreter — costs

evalstone-0.2.0/src/evalstone/center.py

@@ -0,0 +1,253 @@
+"""Attach a project to an Evalstone center (``center:`` in evalstone.yaml).
+
+Attached, the library never holds a provider key: judge (and prompt-target) calls go
+through the center's model proxy, which resolves the model from the project's judge
+settings and dials the provider with the organization's key server-side. The judge
+cache lives in the center (one verdict serves every machine), ``human`` checks look
+labels up there, and every ``evalstone run`` pushes its report. ``providers:`` and
+``judges:`` in the YAML remain the standalone path.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import httpx
+from pydantic import BaseModel
+
+from evalstone.providers import ChatRequest, ChatResponse, ProviderError
+from evalstone.score import MemoryCache, Score
+
+DEFAULT_KEY_ENV = "EVALSTONE_API_KEY"
+
+
+class CenterConfig(BaseModel):
+    url: str
+    project: str  # the center project id (uuid)
+    api_key_env: str = DEFAULT_KEY_ENV  # an ingest key (es_…) or a user token
+    timeout: float = 120.0  # a judge call waits on the provider behind the proxy
+    cache_timeout: float = 10.0  # cache and label lookups: a slow center is a miss, fast
+    push_runs: bool = True  # `evalstone run` pushes its report
+    cache: bool = True  # judge verdicts read/written in the center
+
+    def api_key(self) -> str | None:
+        return os.environ.get(self.api_key_env) or None
+
+
+class CenterError(ProviderError):
+    """The center refused or could not serve a request."""
+
+
+class CenterClient:
+    """Thin HTTP client for the center's project endpoints."""
+
+    def __init__(self, config: CenterConfig, *, client: Any = None) -> None:
+        self.config = config
+        key = config.api_key()
+        if not key:
+            raise CenterError(
+                f"center: no API key — set {config.api_key_env} (an ingest key from the "
+                "project's settings) in the environment"
+            )
+        self._client = client or httpx.Client(
+            base_url=config.url.rstrip("/"),
+            timeout=config.timeout,
+            headers={"Authorization": f"Bearer {key}"},
+        )
+        self._owns = client is None
+
+    def close(self) -> None:
+        if self._owns:
+            self._client.close()
+
+    # ── transport ────────────────────────────────────────────────────────
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> httpx.Response:
+        try:
+            return self._client.request(method, path, **kwargs)
+        except httpx.HTTPError as exc:
+            raise CenterError(f"center: {type(exc).__name__}: {exc}") from exc
+
+    @staticmethod
+    def _detail(resp: httpx.Response) -> str:
+        try:
+            body = resp.json()
+        except ValueError:
+            body = None
+        detail = body.get("detail") if isinstance(body, dict) else body
+        return str(detail)[:300] if detail else resp.text[:300]
+
+    def _json(self, method: str, path: str, **kwargs: Any) -> Any:
+        resp = self._request(method, path, **kwargs)
+        if resp.status_code >= 400:
+            raise CenterError(f"center: HTTP {resp.status_code}: {self._detail(resp)}")
+        try:
+            return resp.json()
+        except ValueError as exc:
+            raise CenterError(f"center: non-JSON response: {resp.text[:200]!r}") from exc
+
+    def _p(self, suffix: str) -> str:
+        return f"/projects/{self.config.project}{suffix}"
+
+    # ── endpoints ────────────────────────────────────────────────────────
+
+    def whoami(self) -> dict[str, Any]:
+        return self._json("GET", "/ingest/whoami")
+
+    def judge_settings(self) -> dict[str, Any]:
+        return self._json("GET", self._p("/judge-settings"))
+
+    def complete(
+        self, model: str | None, request: ChatRequest, *, check_key: str | None = None
+    ) -> ChatResponse:
+        """The proxy call: `model` is `<provider>/<model>` (or None → the center picks)."""
+        body = {
+            "model": model,
+            "check_key": check_key,
+            "system": request.system,
+            "messages": [m.model_dump() for m in request.messages],
+            "max_tokens": request.max_tokens,
+            "temperature": request.temperature,
+        }
+        data = self._json("POST", self._p("/model/complete"), json=body)
+        return ChatResponse.model_validate(data)
+
+    def cache_get(self, key: str) -> Score | None:
+        resp = self._request(
+            "GET", self._p(f"/judge-cache/{key}"), timeout=self.config.cache_timeout
+        )
+        if resp.status_code == 404:
+            return None
+        if resp.status_code >= 400:
+            raise CenterError(f"center: HTTP {resp.status_code}: {self._detail(resp)}")
+        return Score.model_validate(resp.json()["score"])
+
+    def cache_set(self, key: str, score: Score) -> None:
+        judge_version = (
+            (score.raw or {}).get("judge_version") if isinstance(score.raw, dict) else None
+        )
+        self._json(
+            "PUT",
+            self._p(f"/judge-cache/{key}"),
+            json={"score": score.model_dump(mode="json"), "judge_version": judge_version},
+            timeout=self.config.cache_timeout,
+        )
+
+    def label_lookup(
+        self, sample_id: str, output_hash: str, check: str | None
+    ) -> dict[str, Any] | None:
+        params = {"sample_key": sample_id, "output_hash": output_hash}
+        if check is not None:
+            params["check"] = check
+        resp = self._request(
+            "GET", self._p("/labels/lookup"), params=params, timeout=self.config.cache_timeout
+        )
+        if resp.status_code == 404:
+            return None
+        if resp.status_code >= 400:
+            raise CenterError(f"center: HTTP {resp.status_code}: {self._detail(resp)}")
+        return resp.json()
+
+    def push_run(self, report: dict[str, Any]) -> tuple[dict[str, Any], bool]:
+        """Store a run report; returns (run, created) — 200 means the center had it already."""
+        resp = self._request("POST", self._p("/runs"), json=report)
+        if resp.status_code >= 400:
+            raise CenterError(f"center: HTTP {resp.status_code}: {self._detail(resp)}")
+        return resp.json(), resp.status_code == 201
+
+
+# ── the Providers surface, backed by the proxy ───────────────────────────
+
+
+class RemoteProvider:
+    """One named provider entry of the center (a `Provider` whose calls go through the proxy)."""
+
+    def __init__(self, name: str, client: CenterClient) -> None:
+        self.name = name
+        self._client = client
+
+    def complete(self, request: ChatRequest) -> ChatResponse:
+        return self._client.complete(f"{self.name}/{request.model}", request)
+
+
+class CenterProviders:
+    """Drop-in for ``evalstone.providers.Providers`` when a project is attached.
+
+    ``split``/``get``/``complete`` match the registry the judge check and prompt targets
+    call; the default model comes from the center's judge settings, so a check without
+    its own ``model`` resolves there.
+    """
+
+    def __init__(self, client: CenterClient, *, default_model: str | None = None) -> None:
+        self.client = client
+        self.default_model = default_model
+        self._instances: dict[str, RemoteProvider] = {}
+
+    def split(self, model: str | None) -> tuple[str, str]:
+        model = model or self.default_model
+        if not model:
+            raise ProviderError(
+                "no model given and the center project has no default judge model "
+                "(set it in the project's judge settings)"
+            )
+        if "/" not in model:
+            raise ProviderError(f"model must be '<provider>/<model>', got {model!r}")
+        name, _, bare = model.partition("/")
+        return name, bare
+
+    def get(self, name: str) -> RemoteProvider:
+        if name not in self._instances:
+            self._instances[name] = RemoteProvider(name, self.client)
+        return self._instances[name]
+
+    def complete(self, model: str | None, request_without_model: dict[str, Any]) -> ChatResponse:
+        name, bare = self.split(model)
+        req = ChatRequest(model=bare, **request_without_model)
+        return self.client.complete(f"{name}/{bare}", req)
+
+    def close(self) -> None:
+        self.client.close()
+
+
+class CenterCache:
+    """A ``ScoreCache`` in the center, with an in-process layer so a run asks once per key.
+
+    A center hiccup is a miss (a judge then pays for the call); it never fails the run.
+    """
+
+    def __init__(self, client: CenterClient) -> None:
+        self._client = client
+        self._local = MemoryCache()
+        self.errors: list[str] = []
+
+    def get(self, key: str) -> Score | None:
+        hit = self._local.get(key)
+        if hit is not None:
+            return hit
+        try:
+            score = self._client.cache_get(key)
+        except Exception as exc:
+            self.errors.append(f"{type(exc).__name__}: {exc}")
+            return None
+        if score is not None:
+            self._local.set(key, score)
+        return score
+
+    def set(self, key: str, score: Score) -> None:
+        self._local.set(key, score)
+        try:
+            self._client.cache_set(key, score)
+        except Exception as exc:
+            self.errors.append(f"{type(exc).__name__}: {exc}")
+
+
+__all__ = [
+    "DEFAULT_KEY_ENV",
+    "CenterCache",
+    "CenterClient",
+    "CenterConfig",
+    "CenterError",
+    "CenterProviders",
+    "RemoteProvider",
+]

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/checks/_base.py

@@ -151,3 +151,11 @@ def check(
 
 
 __all__ = ["CheckFn", "FunctionCheck", "Registry", "check", "registry"]
+
+
+def judge_model_for(ctx: Any, key: str, model: str | None) -> str | None:
+    """Which model a judge-style check uses: ``--judge-model`` (``ctx.judge_model``) beats the
+    attached project's per-check override (``ctx.extra["judge_models"][key]``) beats the
+    check's own ``model``. ``None`` falls through to the registry's default."""
+    overrides = (getattr(ctx, "extra", None) or {}).get("judge_models") or {}
+    return getattr(ctx, "judge_model", None) or overrides.get(key) or model

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/checks/comparison.py

@@ -11,7 +11,7 @@ from collections.abc import Mapping
 from typing import Any
 
 from evalstone._json import sha256
-from evalstone.checks._base import FunctionCheck
+from evalstone.checks._base import FunctionCheck, judge_model_for
 from evalstone.checks.judge import _json_text, extract_json
 from evalstone.dataset import Sample
 from evalstone.output import Output
@@ -69,7 +69,7 @@ def pairwise(
     def ask(ctx: Context, sample_input: str, a: str, b: str) -> tuple[str, str, float, float]:
         user = USER.format(input=sample_input, a=a, b=b)
         resp = ctx.providers.complete(
-            ctx.judge_model or model,
+            judge_model_for(ctx, key, model),
             {
                 "system": SYSTEM.format(rubric=rubric.strip()),
                 "messages": [ChatMessage(role="user", content=user)],

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/checks/human.py

@@ -85,8 +85,11 @@ def human(
     """Score from a person's label for this (sample, output); unlabelled → error.
 
     ``check`` selects labels recorded for a specific rubric (``evalstone label
-    --check``); ``None`` uses the overall grade.
+    --check``); ``None`` uses the overall grade. ``labels="center"`` (what an attached
+    project gets by default) looks the label up in the center instead of a file.
     """
+    if labels == "center":
+        return FunctionCheck(key, _center_lookup(check), kind=CheckKind.HUMAN)
     store_path = Path(labels)
     cached: dict[str, Any] = {}
 
@@ -121,4 +124,32 @@ def human(
     return FunctionCheck(key, fn, kind=CheckKind.HUMAN)
 
 
+def _center_lookup(check: str | None):
+    def fn(sample: Sample, output: Output, ctx: Any) -> Score:
+        client = (ctx.extra or {}).get("center_client") if ctx is not None else None
+        if client is None:
+            return Score.errored("human: labels=center but the run is not attached to a center")
+        try:
+            found = client.label_lookup(sample.id, output_hash(output), check)
+        except Exception as exc:  # a center hiccup is an error score, never a pass
+            return Score.errored(f"human: center lookup failed: {exc}")
+        if found is None:
+            return Score.errored(
+                f"no human label in the center for {sample.id} @ {output_hash(output)}"
+            )
+        reason = found.get("notes") or ("labelled pass" if found["passed"] else "labelled fail")
+        return Score(
+            value=found["value"],
+            passed=found["passed"],
+            reason=reason,
+            raw={
+                "labeler": found.get("labeler", ""),
+                "grade": found.get("raw"),
+                "at": found.get("labeled_at"),
+            },
+        )
+
+    return fn
+
+
 __all__ = ["Label", "LabelStore", "human", "output_hash"]

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/checks/judge.py

@@ -13,7 +13,7 @@ import re
 from typing import Any
 
 from evalstone._json import dumps, sha256
-from evalstone.checks._base import FunctionCheck
+from evalstone.checks._base import FunctionCheck, judge_model_for
 from evalstone.dataset import Sample
 from evalstone.output import Output
 from evalstone.providers import ChatMessage, ProviderError
@@ -134,7 +134,7 @@ def judge(
         providers = ctx.providers
         if providers is None:
             return Score.errored("judge: no providers configured (ctx.providers is None)")
-        chosen = ctx.judge_model or model  # a run-time override beats the check's own model
+        chosen = judge_model_for(ctx, key, model)
         try:
             resolved = "/".join(providers.split(chosen))
         except ProviderError as exc:

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/cli.py

@@ -14,9 +14,11 @@ from rich.markup import escape
 from rich.table import Table
 
 from evalstone import __version__
+from evalstone.center import CenterError
 from evalstone.config import CONFIG_FILENAME, Config
 from evalstone.providers import ProviderError
 from evalstone.run import DEFAULT_FAILURES, Run, compare
+from evalstone.score import Context
 
 app = typer.Typer(
     name="evalstone",
@@ -136,7 +138,7 @@ def run(
     cfg = _load(config)
     try:
         s, target = cfg.suite(suite)
-    except SUITE_ERRORS as exc:
+    except (*SUITE_ERRORS, CenterError) as exc:
         err.print(f"[red]{escape(str(exc))}[/red]")
         raise typer.Exit(2) from None
     base = _load_run(baseline) if baseline else None
@@ -147,20 +149,48 @@ def run(
         except ProviderError as exc:
             err.print(f"[red]--judge-model: {escape(str(exc))}[/red]")
             raise typer.Exit(2) from None
+    try:
+        ctx = cfg.context(judge_model=judge_model)
+    except CenterError as exc:
+        err.print(f"[red]{escape(str(exc))}[/red]")
+        raise typer.Exit(2) from None
     with console.status(f"running {suite} over {len(s.dataset)} samples…"):
-        result = s.run(
-            target, baseline=base, trigger="cli", ctx=cfg.context(judge_model=judge_model)
-        )
+        result = s.run(target, baseline=base, trigger="cli", ctx=ctx)
     console.print(_run_table(result))
+    _warn_cache_errors(ctx)
     _print_failures(result, failures)
     _print_gate(result)
     out = report or (cfg.root / cfg.report)
     result.to_json(out)
     console.print(f"\nreport → {out}")
+    _push_to_center(cfg, result)
     if strict and result.gate_passed is False:
         raise typer.Exit(1)
 
 
+def _warn_cache_errors(ctx: Context) -> None:
+    """Say so when the center cache was unreachable: those verdicts were paid for."""
+    errors = getattr(ctx.cache, "errors", None)
+    if errors:
+        err.print(
+            f"[yellow]center cache: {len(errors)} request(s) failed "
+            f"(first: {escape(errors[0][:160])}) — judges paid for those verdicts[/yellow]"
+        )
+
+
+def _push_to_center(cfg: Config, result: Run) -> None:
+    """Attached projects store every run in the center; a push failure warns, never fails."""
+    if cfg.center is None or not cfg.center.push_runs:
+        return
+    try:
+        stored, created = cfg.center_client().push_run(result.model_dump(mode="json"))
+    except Exception as exc:
+        err.print(f"[yellow]center: run not pushed — {escape(str(exc))}[/yellow]")
+        return
+    verb = "stored" if created else "already stored"
+    console.print(f"center → run {stored.get('id', '?')} {verb} ({cfg.center.url})")
+
+
 @app.command()
 def gate(
     suite: str = typer.Argument(..., help="Suite name from evalstone.yaml"),

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/config.py

@@ -29,13 +29,14 @@ from typing import Any
 import yaml
 from pydantic import BaseModel, Field, field_validator
 
+from evalstone.center import CenterCache, CenterClient, CenterConfig, CenterProviders
 from evalstone.checks import comparison, deterministic, rag, registry, safety, trajectory
 from evalstone.checks.human import human as human_factory
 from evalstone.checks.judge import judge as judge_factory
 from evalstone.dataset import Dataset
 from evalstone.gate import Gate
 from evalstone.providers import ProviderConfig, Providers
-from evalstone.score import Check, Context
+from evalstone.score import Check, Context, MemoryCache
 from evalstone.suite import Suite
 from evalstone.target import Target
 from evalstone.target import target as make_target
@@ -70,18 +71,51 @@ class Config(BaseModel):
     suites: dict[str, SuiteConfig] = Field(default_factory=dict)
     providers: dict[str, ProviderConfig] = Field(default_factory=dict)
     judges: JudgesConfig = Field(default_factory=JudgesConfig)
+    center: CenterConfig | None = None  # attached: keys, judge model and cache live there
     # model → (USD per 1M input tokens, USD per 1M output tokens); overrides the built-in table
     prices: dict[str, tuple[float, float]] = Field(default_factory=dict)
     report: str = "evalstone-report.json"
     root: Path = Field(default=Path("."), exclude=True)
-    _providers: Providers | None = None
-
-    def provider_registry(self) -> Providers:
-        """One ``Providers`` per config (adapters are created lazily, keys read from env)."""
-        if self._providers is None:
-            self._providers = Providers(
+    _providers: Providers | CenterProviders | None = None
+    _local_providers: Providers | None = None
+    _center_client: CenterClient | None = None
+    _judge_settings: dict[str, Any] | None = None
+
+    @property
+    def attached(self) -> bool:
+        return self.center is not None
+
+    def center_client(self) -> CenterClient:
+        if self.center is None:
+            raise ValueError("this project is not attached to a center (no `center:` block)")
+        if self._center_client is None:
+            self._center_client = CenterClient(self.center)
+        return self._center_client
+
+    def judge_settings(self) -> dict[str, Any]:
+        """The center's judge settings for this project, fetched once per config."""
+        if self._judge_settings is None:
+            self._judge_settings = self.center_client().judge_settings()
+        return self._judge_settings
+
+    def local_provider_registry(self) -> Providers:
+        """The standalone registry: `providers:` + env keys. Prompt targets always use it —
+        the system under test is never routed through the center's judge proxy."""
+        if self._local_providers is None:
+            self._local_providers = Providers(
                 self.providers, default_model=self.judges.default_model, prices=self.prices
             )
+        return self._local_providers
+
+    def provider_registry(self) -> Providers | CenterProviders:
+        """The judges' registry: the center's proxy when attached, else the local one."""
+        if self._providers is None:
+            if self.center is not None:
+                self._providers = CenterProviders(
+                    self.center_client(), default_model=self.judge_settings().get("default_model")
+                )
+            else:
+                self._providers = self.local_provider_registry()
         return self._providers
 
     def context(self, *, judge_model: str | None = None) -> Context:
@@ -89,9 +123,19 @@ class Config(BaseModel):
 
         ``judge_model`` overrides every judge's model for this context only (the
         CLI's ``--judge-model``) — judges and pairwise comparisons, never the target
-        under test: the same suite, a different judge, one flag.
+        under test: the same suite, a different judge, one flag. Attached, the cache is
+        the center's and the project's per-check judge models ride in ``extra``.
         """
-        return Context(providers=self.provider_registry(), judge_model=judge_model)
+        if self.center is None:
+            return Context(providers=self.provider_registry(), judge_model=judge_model)
+        client = self.center_client()
+        settings = self.judge_settings()
+        return Context(
+            providers=self.provider_registry(),
+            judge_model=judge_model,
+            cache=CenterCache(client) if self.center.cache else MemoryCache(),
+            extra={"center_client": client, "judge_models": dict(settings.get("overrides") or {})},
+        )
 
     # ── loading ──────────────────────────────────────────────────────────
 
@@ -138,7 +182,11 @@ class Config(BaseModel):
                 known = ", ".join(sorted(FACTORIES))
                 raise ValueError(f"check {key!r}: unknown type {kind!r} (known: {known})")
             if kind == "human":  # the labels file lives relative to evalstone.yaml
-                spec["labels"] = self.root / spec.get("labels", "labels.jsonl")
+                labels = spec.get("labels")
+                if labels == "center" or (labels is None and self.center is not None):
+                    spec["labels"] = "center"  # attached: labels live in the center
+                else:
+                    spec["labels"] = self.root / (labels or "labels.jsonl")
             baseline = spec.get("baseline")
             if kind == "comparison.pairwise" and isinstance(baseline, dict) and "kind" in baseline:
                 spec["baseline"] = self.target(baseline)  # a target spec → a Target
@@ -162,7 +210,7 @@ class Config(BaseModel):
                     "prompt target needs its own `model` — it is the system under test and must "
                     "not silently share judges.default_model (a model grading itself)"
                 )
-            spec["providers"] = self.provider_registry()
+            spec["providers"] = self.local_provider_registry()
         return make_target(kind, **spec)
 
     def suite(self, name: str) -> tuple[Suite, Target]:

{evalstone-0.1.1 → evalstone-0.2.0}/src/evalstone/mcp_server.py

@@ -74,6 +74,8 @@ class Session:
 
     def config(self) -> Config:
         cfg = Config.load(self.config_path or CONFIG_FILENAME)
+        if cfg.attached:
+            return cfg  # judges go through the center's proxy; targets use cfg's local registry
         key = canonical(
             {
                 "providers": {k: v.model_dump(mode="json") for k, v in cfg.providers.items()},
@@ -91,6 +93,8 @@ class Session:
         return cfg
 
     def context(self, cfg: Config) -> Context:
+        if cfg.attached:
+            return cfg.context()  # center proxy, center cache, per-check overrides, labels
         return Context(providers=cfg.provider_registry(), cache=self.cache)
 
     def root(self) -> Path:

evalstone-0.2.0/tests/test_center.py

@@ -0,0 +1,343 @@
+"""The center attach: the proxy as a Providers, the center as a ScoreCache, labels, run push."""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+import respx
+
+from evalstone.center import CenterCache, CenterClient, CenterConfig, CenterError, CenterProviders
+from evalstone.checks.human import human, output_hash
+from evalstone.checks.judge import judge
+from evalstone.config import Config
+from evalstone.dataset import Sample
+from evalstone.output import Output
+from evalstone.providers import ChatMessage, ProviderError
+from evalstone.score import Context, Score
+
+URL = "http://center.test"
+PROJECT = "11111111-2222-4333-8444-555555555555"
+P = f"{URL}/projects/{PROJECT}"
+
+
+@pytest.fixture
+def cfg(monkeypatch):
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    return CenterConfig(url=URL + "/", project=PROJECT)
+
+
+def _verdict(text='{"score": 4, "reason": "fine"}', model="small"):
+    return httpx.Response(
+        200,
+        json={
+            "text": text,
+            "model": model,
+            "input_tokens": 10,
+            "output_tokens": 2,
+            "cost_usd": 0.00001,
+            "latency_ms": 12.5,
+            "stop_reason": "stop",
+        },
+    )
+
+
+def test_client_needs_a_key(monkeypatch):
+    monkeypatch.delenv("EVALSTONE_API_KEY", raising=False)
+    with pytest.raises(CenterError, match="EVALSTONE_API_KEY"):
+        CenterClient(CenterConfig(url=URL, project=PROJECT))
+
+
+@respx.mock
+def test_providers_complete_posts_the_proxy_body_with_the_bearer(cfg):
+    route = respx.post(f"{P}/model/complete").mock(return_value=_verdict())
+    providers = CenterProviders(CenterClient(cfg), default_model="judge/small")
+    resp = providers.complete(
+        None,
+        {"system": "Grade.", "messages": [ChatMessage(role="user", content="x")], "max_tokens": 5},
+    )
+    assert resp.text.startswith('{"score": 4') and resp.cost_usd == pytest.approx(0.00001)
+    sent = route.calls[0].request
+    assert sent.headers["Authorization"] == "Bearer es_test_key"
+    body = json.loads(sent.content)
+    assert body["model"] == "judge/small" and body["system"] == "Grade." and body["max_tokens"] == 5
+    assert body["messages"] == [{"role": "user", "content": "x"}]
+    assert providers.split("other/big") == ("other", "big")
+    with pytest.raises(ProviderError, match="<provider>/<model>"):
+        providers.split("bare")
+    with pytest.raises(ProviderError, match="default judge model"):
+        CenterProviders(CenterClient(cfg)).split(None)
+
+
+@respx.mock
+def test_center_errors_become_provider_errors(cfg):
+    respx.post(f"{P}/model/complete").mock(
+        return_value=httpx.Response(502, json={"detail": "provider 'judge' failed: HTTP 401"})
+    )
+    providers = CenterProviders(CenterClient(cfg))
+    with pytest.raises(ProviderError, match="HTTP 502: provider 'judge' failed"):
+        providers.complete("judge/small", {"messages": [ChatMessage(role="user", content="x")]})
+    respx.post(f"{P}/model/complete").mock(side_effect=httpx.ConnectError("refused"))
+    with pytest.raises(ProviderError, match="ConnectError"):
+        providers.complete("judge/small", {"messages": [ChatMessage(role="user", content="x")]})
+
+
+@respx.mock
+def test_cache_get_set_and_tolerance(cfg):
+    key = "a" * 64
+    get = respx.get(f"{P}/judge-cache/{key}").mock(return_value=httpx.Response(404))
+    put = respx.put(f"{P}/judge-cache/{key}").mock(return_value=httpx.Response(201, json={}))
+    cache = CenterCache(CenterClient(cfg))
+    assert cache.get(key) is None
+    score = Score(value=0.75, passed=True, reason="4/5", raw={"score": 4, "judge_version": "j1"})
+    cache.set(key, score)
+    assert json.loads(put.calls[0].request.content)["judge_version"] == "j1"
+    assert cache.get(key) == score and get.call_count == 1  # the local layer answers now
+    other = "b" * 64
+    respx.get(f"{P}/judge-cache/{other}").mock(
+        return_value=httpx.Response(200, json={"score": score.model_dump(mode="json")})
+    )
+    assert cache.get(other) == score
+    down = "c" * 64
+    respx.get(f"{P}/judge-cache/{down}").mock(side_effect=httpx.ConnectError("down"))
+    assert cache.get(down) is None and cache.errors  # a hiccup is a miss, not a failure
+
+
+@respx.mock
+def test_judge_through_the_center_uses_overrides_and_caches_there(cfg):
+    respx.get(f"{P}/judge-cache/__").mock(return_value=httpx.Response(404))
+    respx.route(method="GET", path__regex=r"/judge-cache/.*").mock(return_value=httpx.Response(404))
+    puts = respx.route(method="PUT", path__regex=r"/judge-cache/.*").mock(
+        return_value=httpx.Response(201, json={})
+    )
+    calls = respx.post(f"{P}/model/complete").mock(return_value=_verdict())
+    client = CenterClient(cfg)
+    ctx = Context(
+        providers=CenterProviders(client, default_model="judge/small"),
+        cache=CenterCache(client),
+        extra={"center_client": client, "judge_models": {"strict": "other/big"}},
+    )
+    check = judge("strict", "Be strict.")
+    score = check.evaluate(Sample(input="q"), Output(value="a"), ctx)
+    assert score.passed and score.value == 0.75
+    assert json.loads(calls.calls[0].request.content)["model"] == "other/big"  # the override
+    assert puts.call_count == 1  # the verdict went to the center
+    assert judge("plain", "Be nice.").evaluate(Sample(input="q"), Output(value="a"), ctx).passed
+    assert json.loads(calls.calls[1].request.content)["model"] == "judge/small"  # the default
+    ctx2 = ctx.model_copy(update={"judge_model": "cli/flag"})
+    judge("strict", "Be strict.").evaluate(Sample(input="q"), Output(value="a"), ctx2)
+    assert json.loads(calls.calls[2].request.content)["model"] == "cli/flag"  # --judge-model wins
+
+
+@respx.mock
+def test_human_labels_from_the_center(cfg):
+    client = CenterClient(cfg)
+    ctx = Context(extra={"center_client": client})
+    sample, out = Sample(id="s1", input="q"), Output(value="the answer")
+    route = respx.get(f"{P}/labels/lookup").mock(return_value=httpx.Response(404))
+    check = human(labels="center", check="helpful")
+    missing = check.evaluate(sample, out, ctx)
+    assert missing.error and "no human label in the center" in missing.error
+    assert route.calls[0].request.url.params["output_hash"] == output_hash(out)
+    assert route.calls[0].request.url.params["check"] == "helpful"
+    respx.get(f"{P}/labels/lookup").mock(
+        return_value=httpx.Response(
+            200, json={"value": 1.0, "passed": True, "notes": "good", "labeler": "ada", "raw": 5}
+        )
+    )
+    got = check.evaluate(sample, out, ctx)
+    assert got.passed and got.reason == "good" and got.raw["labeler"] == "ada"
+    unattached = human(labels="center").evaluate(sample, out, Context())
+    assert unattached.error and "not attached" in unattached.error
+
+
+@respx.mock
+def test_config_attached_wires_registry_cache_and_human(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    respx.get(f"{P}/judge-settings").mock(
+        return_value=httpx.Response(
+            200,
+            json={"project_id": PROJECT, "default_model": "judge/small", "overrides": {"h": "o/b"}},
+        )
+    )
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\n"
+        "checks:\n  h:\n    type: human\n  f:\n    type: human\n    labels: my.jsonl\n"
+    )
+    cfg = Config.load(tmp_path)
+    assert cfg.attached
+    ctx = cfg.context()
+    assert (
+        isinstance(ctx.providers, CenterProviders) and ctx.providers.default_model == "judge/small"
+    )
+    assert isinstance(ctx.cache, CenterCache)
+    assert (
+        ctx.extra["judge_models"] == {"h": "o/b"}
+        and ctx.extra["center_client"] is cfg.center_client()
+    )
+    assert cfg.check("h").evaluate(Sample(id="s", input="q"), Output(value="a"), Context()).error
+    file_mode = cfg.check("f").evaluate(Sample(id="s", input="q"), Output(value="a"), Context())
+    assert "my.jsonl" in file_mode.error
+    assert Config.model_validate({"project": "x"}).attached is False
+
+
+@respx.mock
+def test_cli_run_pushes_the_report(tmp_path, monkeypatch):
+    from typer.testing import CliRunner
+
+    from evalstone.cli import app
+
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    respx.get(f"{P}/judge-settings").mock(
+        return_value=httpx.Response(
+            200, json={"project_id": PROJECT, "default_model": None, "overrides": {}}
+        )
+    )
+    pushed = respx.post(f"{P}/runs").mock(return_value=httpx.Response(201, json={"id": "run-1"}))
+    (tmp_path / "golden.jsonl").write_text('{"input": "hi", "expected": "hi"}\n')
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\n"
+        "datasets:\n  golden: golden.jsonl\n"
+        "checks:\n  same:\n    type: exact\n"
+        "suites:\n  ci:\n    dataset: golden\n"
+        "    target: {kind: recorded, outputs: {}}\n    checks: [same]\n"
+    )
+    monkeypatch.chdir(tmp_path)
+    result = CliRunner().invoke(app, ["run", "ci"])
+    assert result.exit_code == 0, result.output
+    assert "center → run run-1 stored" in result.output
+    body = json.loads(pushed.calls[0].request.content)
+    assert body["suite"] == "ci" and body["results"]
+    pushed.mock(return_value=httpx.Response(500, json={"detail": "boom"}))
+    result = CliRunner().invoke(app, ["run", "ci"])
+    assert result.exit_code == 0  # a failed push never fails the run
+    assert "run not pushed" in result.output + getattr(result, "stderr", "")
+
+
+# --- regressions from the #33 review ---------------------------------------------------------
+
+
+@respx.mock
+def test_explicit_labels_center_in_yaml_reaches_the_center(tmp_path, monkeypatch):
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    respx.get(f"{P}/judge-settings").mock(
+        return_value=httpx.Response(
+            200, json={"project_id": PROJECT, "default_model": None, "overrides": {}}
+        )
+    )
+    lookup = respx.get(f"{P}/labels/lookup").mock(return_value=httpx.Response(404))
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\n"
+        "checks:\n  h:\n    type: human\n    labels: center\n"
+    )
+    cfg = Config.load(tmp_path)
+    score = cfg.check("h").evaluate(Sample(id="s", input="q"), Output(value="a"), cfg.context())
+    assert "in the center" in score.error and lookup.call_count == 1
+
+
+@respx.mock
+def test_mcp_session_honours_the_attach(tmp_path, monkeypatch):
+    from evalstone.mcp_server import Session
+
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    respx.get(f"{P}/judge-settings").mock(
+        return_value=httpx.Response(
+            200, json={"project_id": PROJECT, "default_model": "j/m", "overrides": {}}
+        )
+    )
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\n"
+    )
+    session = Session(config_path=str(tmp_path / "evalstone.yaml"))
+    cfg = session.config()
+    ctx = session.context(cfg)
+    assert isinstance(ctx.providers, CenterProviders) and isinstance(ctx.cache, CenterCache)
+    assert ctx.extra["center_client"] is cfg.center_client()
+
+
+def test_cli_center_error_while_building_the_suite_is_a_clean_exit(tmp_path, monkeypatch):
+    from typer.testing import CliRunner
+
+    from evalstone.cli import app
+
+    monkeypatch.delenv("EVALSTONE_API_KEY", raising=False)
+    (tmp_path / "golden.jsonl").write_text('{"input": "hi"}\n')
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\n"
+        "datasets:\n  golden: golden.jsonl\n"
+        "checks:\n  j:\n    type: judge\n    rubric: ok?\n"
+        "suites:\n  ci:\n    dataset: golden\n"
+        "    target: {kind: prompt, model: openai/x, template: '{input}'}\n    checks: [j]\n"
+    )
+    monkeypatch.chdir(tmp_path)
+    result = CliRunner().invoke(app, ["run", "ci"])
+    assert result.exit_code == 2 and "EVALSTONE_API_KEY" in result.output + getattr(
+        result, "stderr", ""
+    )
+    assert "Traceback" not in result.output
+
+
+@respx.mock
+def test_cache_treats_a_corrupt_row_and_a_non_object_error_as_misses(cfg):
+    key = "d" * 64
+    respx.get(f"{P}/judge-cache/{key}").mock(
+        return_value=httpx.Response(200, json={"score": {"value": 1.2, "passed": True}})  # invalid
+    )
+    cache = CenterCache(CenterClient(cfg))
+    assert cache.get(key) is None and "ValidationError" in cache.errors[0]
+    other = "e" * 64
+    respx.get(f"{P}/judge-cache/{other}").mock(
+        return_value=httpx.Response(504, json=["upstream timeout"])
+    )
+    assert cache.get(other) is None and "upstream timeout" in cache.errors[1]
+
+
+@respx.mock
+def test_pairwise_honours_the_center_override(cfg):
+    from evalstone.checks.comparison import pairwise
+
+    calls = respx.post(f"{P}/model/complete").mock(
+        return_value=_verdict(text='{"winner": "A", "reason": "clearer"}')
+    )
+    client = CenterClient(cfg)
+    ctx = Context(
+        providers=CenterProviders(client, default_model="judge/small"),
+        extra={"center_client": client, "judge_models": {"better": "other/big"}},
+    )
+    check = pairwise(
+        "better", "Which is clearer?", baseline={"s": "baseline answer"}, model="judge/small"
+    )
+    check.evaluate(Sample(id="s", input="q"), Output(value="candidate"), ctx)
+    assert json.loads(calls.calls[0].request.content)["model"] == "other/big"
+
+
+@respx.mock
+def test_prompt_targets_stay_local_when_attached(tmp_path, monkeypatch):
+    from evalstone.providers import Providers
+
+    monkeypatch.setenv("EVALSTONE_API_KEY", "es_test_key")
+    respx.get(f"{P}/judge-settings").mock(
+        return_value=httpx.Response(
+            200, json={"project_id": PROJECT, "default_model": None, "overrides": {}}
+        )
+    )
+    (tmp_path / "evalstone.yaml").write_text(
+        f"project: demo\ncenter:\n  url: {URL}\n  project: {PROJECT}\nproviders:\n  openai: {{}}\n"
+    )
+    cfg = Config.load(tmp_path)
+    target = cfg.target({"kind": "prompt", "model": "openai/x", "template": "{input}"})
+    assert isinstance(target.providers, Providers)  # the local registry, not the proxy
+    assert isinstance(cfg.provider_registry(), CenterProviders)  # judges: the proxy
+
+
+@respx.mock
+def test_cache_lookups_use_the_short_timeout_and_the_cli_warns(cfg):
+    key = "f" * 64
+    route = respx.get(f"{P}/judge-cache/{key}").mock(side_effect=httpx.ReadTimeout("slow"))
+    cache = CenterCache(CenterClient(cfg))
+    assert cache.get(key) is None
+    assert route.calls[0].request.extensions["timeout"]["read"] == cfg.cache_timeout
+    from evalstone.cli import _warn_cache_errors
+
+    _warn_cache_errors(Context(cache=cache))  # prints a warning; must not raise