agentx-python 0.6.35__tar.gz → 0.6.37__tar.gz

{agentx_python-0.6.35 → agentx_python-0.6.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.35
+Version: 0.6.37
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -315,6 +315,21 @@ evaluator = client.monitor.online_evaluators.builder(
 client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
 ```
 
+An online evaluator and a dataset run's grading config are two profiles of one **LLM Judge Scorer**, and `client.monitor.judge_scorers` manages both in a single call - one rubric, one id, used offline and online:
+
+```python
+scorer = client.monitor.judge_scorers.builder(
+    name="Helpfulness",
+    acceptance_criteria="Concrete, correct, cites the policy.",
+    live=True, sample_rate=0.1, alert_threshold=5,   # the online profile
+).publish()
+
+client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=scorer.id)
+client.monitor.judge_scorers.ratings(scorer.id, window="7d")
+```
+
+The two clients above are that entity's per-profile legacy views; they keep working and share the same ids. See [LLM Judge Scorers](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs) for the full surface, including which self-host engine builds serve it.
+
 An online evaluator can also judge **whole conversations** instead of single traces: pass `scope="session"` and the engine scores each multi-turn session once it's been idle for `idle_seconds`, re-scoring if the conversation resumes. And to close the loop with reality, two ground-truth streams feed the dashboard's Judge Calibration view (which measures how often AgentX's automated verdicts agree with what actually happened): `client.outcomes.report(...)` for after-the-fact system results (a reopened ticket, a human confirmation), and `client.feedback.report(...)` for end-user votes forwarded from your own app's UI - a "down" raises a "Negative user feedback" signal directly, no sampling or judge call involved. All self-host features.
 
 ```python

{agentx_python-0.6.35 → agentx_python-0.6.37}/README.md

@@ -251,6 +251,21 @@ evaluator = client.monitor.online_evaluators.builder(
 client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
 ```
 
+An online evaluator and a dataset run's grading config are two profiles of one **LLM Judge Scorer**, and `client.monitor.judge_scorers` manages both in a single call - one rubric, one id, used offline and online:
+
+```python
+scorer = client.monitor.judge_scorers.builder(
+    name="Helpfulness",
+    acceptance_criteria="Concrete, correct, cites the policy.",
+    live=True, sample_rate=0.1, alert_threshold=5,   # the online profile
+).publish()
+
+client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=scorer.id)
+client.monitor.judge_scorers.ratings(scorer.id, window="7d")
+```
+
+The two clients above are that entity's per-profile legacy views; they keep working and share the same ids. See [LLM Judge Scorers](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs) for the full surface, including which self-host engine builds serve it.
+
 An online evaluator can also judge **whole conversations** instead of single traces: pass `scope="session"` and the engine scores each multi-turn session once it's been idle for `idle_seconds`, re-scoring if the conversation resumes. And to close the loop with reality, two ground-truth streams feed the dashboard's Judge Calibration view (which measures how often AgentX's automated verdicts agree with what actually happened): `client.outcomes.report(...)` for after-the-fact system results (a reopened ticket, a human confirmation), and `client.feedback.report(...)` for end-user votes forwarded from your own app's UI - a "down" raises a "Negative user feedback" signal directly, no sampling or judge call involved. All self-host features.
 
 ```python

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/agentx.py

@@ -94,8 +94,18 @@ class AgentX:
 
     @classmethod
     def from_env(cls) -> "AgentX":
-        """Create an AgentX client using AGENTX_API_KEY (and optionally AGENTX_API_BASE_URL) from the environment."""
-        return cls()
+        """Create an AgentX client from the environment: AGENTX_API_KEY plus, for the base URL,
+        the first of AGENTX_API_BASE_URL / AGENTX_SELFHOST_BASE_URL / BASE_URL that is set.
+        The fallbacks match the conventions the self-host samples and .env files already use,
+        so from_env works wherever an explicit AgentX(base_url=...) would."""
+        import os
+
+        base_url = (
+            os.getenv("AGENTX_API_BASE_URL")
+            or os.getenv("AGENTX_SELFHOST_BASE_URL")
+            or os.getenv("BASE_URL")
+        )
+        return cls(base_url=base_url) if base_url else cls()
 
     def get_agent(self, id: str) -> Agent:
         url = f"{self.base_url or api_base()}/access/agents/{id}"

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/evaluations/client.py

@@ -59,6 +59,15 @@ class AgentXValidationError(AgentXEvaluationsError):
     pass
 
 
+def _resolve_scorer_id(scorer_id: Optional[str], evaluation_settings_id: Optional[str]) -> Optional[str]:
+    """One grader, two spellings: ``scorer_id`` is the post-consolidation name for what the wire
+    still calls ``evaluationSettingsId`` (the ids are identical by design). Both kwargs are
+    accepted everywhere a run picks its grader; passing both with different values is a bug."""
+    if scorer_id and evaluation_settings_id and scorer_id != evaluation_settings_id:
+        raise ValueError("Pass either scorer_id or evaluation_settings_id (they are the same id), not two different ids")
+    return scorer_id or evaluation_settings_id
+
+
 class EvaluationsClient:
     def __init__(
         self,
@@ -100,11 +109,21 @@ class EvaluationsClient:
         from agentx.evaluations.prompts import PromptClient
 
         self.datasets = DatasetClient(self)
-        self.settings = EvaluationSettingsClient(self)
+        # Legacy view of an LLM Judge Scorer's offline profile - constructed lazily so its
+        # DeprecationWarning fires on first USE, not for every client that never touches it.
+        self._settings: "EvaluationSettingsClient | None" = None
         self.prompts = PromptClient(self)
         from agentx.evaluations.tool_schemas import ToolSchemaClient
         self.tool_schemas = ToolSchemaClient(self)
 
+    @property
+    def settings(self) -> "EvaluationSettingsClient":
+        if self._settings is None:
+            from agentx.evaluations.evaluation_settings import EvaluationSettingsClient
+
+            self._settings = EvaluationSettingsClient(self)
+        return self._settings
+
     # ------------------------------------------------------------------
     # Low-level HTTP
     # ------------------------------------------------------------------
@@ -275,10 +294,16 @@ class EvaluationsClient:
         dataset_id: str,
         subject: EvaluationSubject,
         python_version: Optional[str] = None,
+        scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRun:
+        """``scorer_id`` names the LLM Judge Scorer grading this run (its id doubles as the
+        wire's ``evaluationSettingsId``). ``evaluation_settings_id`` is the pre-consolidation
+        alias and keeps working."""
         from agentx.version import VERSION
 
+        grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
+
         payload = {
             "datasetId": dataset_id,
             "evaluationSubject": subject.model_dump(by_alias=True, exclude_none=True),
@@ -290,8 +315,8 @@ class EvaluationsClient:
                 "pythonVersion": python_version or _python_version(),
             },
         }
-        if evaluation_settings_id:
-            payload["evaluationSettingsId"] = evaluation_settings_id
+        if grader_id:
+            payload["evaluationSettingsId"] = grader_id
         data = self._request("POST", "/runs", json=self._with_workspace(payload))
         return EvaluationRun(**data)
 

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/evaluations/datasets.py

@@ -290,6 +290,7 @@ class DatasetClient:
         rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
     ) -> DatasetBuilder:
         return DatasetBuilder(
             self._client,
@@ -307,6 +308,9 @@ class DatasetClient:
             rouge_score=rouge_score,
             similarity_model=similarity_model,
             sovereignty_models=sovereignty_models,
+            # Was documented (evaluation/code-scorers.mdx) but not forwarded - DatasetBuilder
+            # itself always accepted it. Fixed with the judge-scorer unification.
+            code_scorers=code_scorers,
         )
 
     def from_csv(self, path: str, name: str, **kwargs) -> DatasetBuilder:

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/evaluations/evaluation_settings.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
 from agentx.evaluations.models import EvaluationSettings
@@ -81,10 +82,25 @@ class EvaluationSettingsBuilder:
 
 
 class EvaluationSettingsClient:
-    """Thin wrapper surfaced as client.evaluations.settings."""
+    """Thin wrapper surfaced as ``client.evaluations.settings``.
+
+    Note: an evaluation-settings record is the judge rubric + OFFLINE profile of an
+    **LLM Judge Scorer** - the unified entity at ``client.monitor.judge_scorers``, which also
+    carries the optional online (live-traffic) profile. This client keeps working unchanged;
+    prefer ``judge_scorers`` for new code so both profiles live in one place -
+    ``client.monitor.judge_scorers.builder(...)`` has the same snake_case ergonomics as the
+    builder below, plus tool_context, thresholds, and the live profile in one call."""
 
     def __init__(self, client: "EvaluationsClient"):
         self._client = client
+        # Soft deprecation: hidden by default (DeprecationWarning), visible under -W or pytest.
+        warnings.warn(
+            "client.evaluations.settings is the legacy view of an LLM Judge Scorer's offline "
+            "profile; prefer client.monitor.judge_scorers, which manages the judge rubric, "
+            "offline profile, and online (live-traffic) profile as one entity.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
 
     def builder(
         self,

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/evaluations/models.py

@@ -94,7 +94,7 @@ class Dataset(BaseModel):
 
 class EvaluationSettings(BaseModel):
     """A standalone, reusable grading config - no dataset/questions attached.
-    Created via ``client.evaluations.settings.builder(...).publish()`` and run
+    Created via ``client.monitor.judge_scorers.builder(...).publish()`` (or the legacy ``client.evaluations.settings.builder``) and run
     against any dataset by passing its id as ``evaluation_settings_id`` to
     ``client.evaluations.run(...)``."""
 

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/evaluations/runner.py

@@ -421,10 +421,15 @@ class EvaluationsRunner:
     def __init__(self, client: EvaluationsClient):
         self._client = client
         self.datasets = client.datasets
-        self.settings = client.settings
         self.prompts = client.prompts
         self.tool_schemas = client.tool_schemas
 
+    @property
+    def settings(self):
+        # Deferred: touching client.settings eagerly here would fire its legacy-surface
+        # DeprecationWarning for every runner, used or not.
+        return self._client.settings
+
     def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
         """List the LLM models AgentX supports - the same set selectable for
         the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
@@ -482,24 +487,24 @@ class EvaluationsRunner:
         self,
         dataset_id: str,
         subject: Union[Dict[str, Any], EvaluationSubject],
+        scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRunContext:
-        """Start a run of ``dataset_id`` against ``subject``. Pass
-        ``evaluation_settings_id`` to grade against a standalone, reusable
-        config (created via ``client.evaluations.settings.builder(...)``)
-        instead of the dataset's own default config."""
+        """Start a run of ``dataset_id`` against ``subject``. Pass ``scorer_id`` (an LLM Judge
+        Scorer's id, e.g. from ``client.monitor.judge_scorers``) to grade with a specific
+        scorer instead of the dataset's default. ``evaluation_settings_id`` is the
+        pre-consolidation alias for the same id and keeps working."""
+        from agentx.evaluations.client import _resolve_scorer_id
+
         if isinstance(subject, dict):
             subject = EvaluationSubject(**subject)
 
+        grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
         dataset = self._client.get_dataset(dataset_id)
         evaluation_settings = (
-            self._client.get_evaluation_settings(evaluation_settings_id)
-            if evaluation_settings_id
-            else None
-        )
-        run = self._client.init_run(
-            dataset_id, subject, evaluation_settings_id=evaluation_settings_id
+            self._client.get_evaluation_settings(grader_id) if grader_id else None
         )
+        run = self._client.init_run(dataset_id, subject, scorer_id=grader_id)
         logger.info(
             "Started evaluation run %s on dataset %s (%d case(s), %d repetition(s))",
             run.run_id,

{agentx_python-0.6.35 → agentx_python-0.6.37}/agentx/monitor/client.py

@@ -88,8 +88,15 @@ class MonitorClient:
         # Handed this client's own resolved API root, never the process-global default, so a
         # second AgentX() with a different base_url can't re-point it (deep-dive bug #1).
         self.scorers = ScorersClient(api_key=api_key, base_url=self._api_root())
+        from agentx.monitor.judge_scorers import JudgeScorersClient
+        # The unified LLM Judge Scorer (rubric + offline/online profiles in one entity) - the
+        # surface that matches the product; evaluations.settings and online_evaluators below
+        # remain as its profile-level views.
+        self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root())
         self.profile = MonitorProfileClient(self)
-        self.online_evaluators = MonitorOnlineEvaluatorClient(self)
+        # Legacy view of an LLM Judge Scorer's online profile - constructed lazily so its
+        # DeprecationWarning fires on first USE, not for every client that never touches it.
+        self._online_evaluators: "MonitorOnlineEvaluatorClient | None" = None
         from agentx.monitor.sessions import MonitorSessionClient
         from agentx.monitor.agents import MonitorAgentClient
         self.sessions = MonitorSessionClient(self)
@@ -173,6 +180,14 @@ class MonitorClient:
         data = self._request("POST", "/online-evaluators", json=self._with_workspace(payload))
         return MonitorOnlineEvaluator(**data["evaluator"])
 
+    @property
+    def online_evaluators(self) -> "MonitorOnlineEvaluatorClient":
+        if self._online_evaluators is None:
+            from agentx.monitor.online_evaluators import MonitorOnlineEvaluatorClient
+
+            self._online_evaluators = MonitorOnlineEvaluatorClient(self)
+        return self._online_evaluators
+
     def list_online_evaluators(self) -> List[MonitorOnlineEvaluator]:
         data = self._request("GET", "/online-evaluators", params=self._workspace_params())
         return [MonitorOnlineEvaluator(**e) for e in data.get("evaluators", [])]

agentx_python-0.6.37/agentx/monitor/judge_scorers.py

@@ -0,0 +1,334 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+_SENTINEL: Any = object()
+
+
+class AgentXJudgeScorersError(Exception):
+    pass
+
+
+class JudgeScorer(dict):
+    """Wire object for the unified LLM Judge Scorer (dict subclass so unknown fields
+    round-trip). Convenience properties expose the pieces scripts reach for most."""
+
+    @property
+    def id(self) -> str:
+        return self["_id"]
+
+    @property
+    def name(self) -> str:
+        return self["name"]
+
+    @property
+    def judge(self) -> Dict[str, Any]:
+        return self.get("judge", {})
+
+    @property
+    def offline(self) -> Dict[str, Any]:
+        return self.get("offline", {})
+
+    @property
+    def online(self) -> Optional[Dict[str, Any]]:
+        return self.get("online")
+
+    @property
+    def online_profile_id(self) -> Optional[str]:
+        online = self.online
+        return online.get("profileId") if online else None
+
+
+class JudgeScorersClient:
+    """Surfaced as ``client.monitor.judge_scorers``: the unified LLM Judge Scorer.
+
+    One scorer = one judge rubric (``judge``: acceptance/rejection/evaluation criteria, judge
+    prompt, judge model) + two setting profiles:
+
+    - ``offline`` - how dataset runs grade with it (repetitions, similarity metrics, code
+      scorers, default/status). The scorer's ``id`` is exactly what
+      ``client.evaluations.run(dataset_id, scorer_id=scorer.id)`` takes.
+    - ``online`` - whether/how it scores live traffic (enabled, sample rate, scope,
+      alert threshold). ``None`` means offline-only.
+
+    Strictly one online profile per scorer. This supersedes the split between
+    ``client.evaluations.settings`` (the offline half) and ``client.monitor.online_evaluators``
+    (the online half) - both keep working, but this is the surface that matches the product.
+
+    Example::
+
+        scorer = client.monitor.judge_scorers.create(
+            "Support quality",
+            judge={"acceptanceCriteria": "Concrete, correct, cites the policy."},
+            online={"enabled": True, "sampleRate": 0.2, "alertThreshold": 6},
+        )
+        client.evaluations.run(dataset_id, subject, scorer_id=scorer.id)
+        cal = client.monitor.judge_scorers.calibration(scorer.id)
+    """
+
+    def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None):
+        self._api_key = api_key
+        # Captured once at construction so two clients with different bases can coexist.
+        self._base_url = (base_url or api_base()).rstrip("/")
+
+    def _request(self, method: str, path: str, json: Any = None, timeout: int = 60) -> Any:
+        resp = requests.request(
+            method,
+            f"{self._base_url}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=timeout,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXJudgeScorersError(f"Judge scorer request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    # ------------------------------------------------------------------
+    # CRUD
+    # ------------------------------------------------------------------
+
+    def builder(
+        self,
+        name: str,
+        description: Optional[str] = None,
+        *,
+        # Judge rubric (shared by every surface that runs this scorer)
+        acceptance_criteria: Optional[str] = None,
+        rejection_criteria: Optional[str] = None,
+        evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
+        tool_context: Optional[str] = None,
+        # Offline profile (dataset-run grading)
+        number_of_requests: int = 1,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
+        thresholds: Optional[Dict[str, Any]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
+        is_default: bool = False,
+        # Online profile (live traffic). live=True enables scoring immediately - every check is
+        # a real judge call on your own key.
+        live: bool = False,
+        sample_rate: float = 0.1,
+        scope: str = "trace",
+        alert_threshold: Optional[float] = 5,
+        severity: str = "medium",
+        agent_ids: Optional[List[str]] = None,
+        idle_seconds: int = 120,
+    ) -> "JudgeScorerBuilder":
+        """Snake_case builder with ``.publish()``, the unified successor of
+        ``client.evaluations.settings.builder(...)`` - same offline fields (plus ``thresholds``,
+        ``tool_context``) and, new here, the online profile in the same call. The scorer the
+        builder publishes is one entity: its ``.id`` is what ``client.evaluations.run(...,
+        scorer_id=...)`` takes, and its live profile is what online scoring keys on."""
+        judge: Dict[str, Any] = {}
+        for key, value in (
+            ("acceptanceCriteria", acceptance_criteria),
+            ("rejectionCriteria", rejection_criteria),
+            ("evaluationCriteria", evaluation_criteria),
+            ("judgePrompt", judge_prompt),
+            ("judgeModel", judge_model),
+            ("toolContext", tool_context),
+        ):
+            if value is not None:
+                judge[key] = value
+        offline: Dict[str, Any] = {"numberOfRequests": number_of_requests, "isDefault": is_default}
+        if vector_similarity:
+            offline["vectorSimilarity"] = (
+                {"enabled": True, "model": similarity_model} if similarity_model else {"enabled": True}
+            )
+        if jaccard_similarity:
+            offline["jaccardSimilarity"] = {"enabled": True}
+        if bleu_score:
+            offline["bleuScore"] = {"enabled": True}
+        if rouge_score:
+            offline["rougeScore"] = {"enabled": True}
+        if sovereignty_models is not None:
+            offline["sovereigntyIndex"] = {"enabled": bool(sovereignty_models), "models": sovereignty_models}
+        if thresholds is not None:
+            offline["thresholds"] = thresholds
+        if code_scorers is not None:
+            offline["codeScorers"] = code_scorers
+        online: Optional[Dict[str, Any]] = None
+        if live:
+            online = {
+                "enabled": True,
+                "sampleRate": sample_rate,
+                "scope": scope,
+                "alertThreshold": alert_threshold,
+                "severity": severity,
+                "idleSeconds": idle_seconds,
+            }
+            if agent_ids:
+                online["scopeMode"] = "selected"
+                online["agentIds"] = agent_ids
+        return JudgeScorerBuilder(self, name=name, description=description, judge=judge, offline=offline, online=online)
+
+    def create(
+        self,
+        name: str,
+        *,
+        description: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        offline: Optional[Dict[str, Any]] = None,
+        online: Optional[Dict[str, Any]] = None,
+    ) -> JudgeScorer:
+        """Create a scorer. ``online=None`` (default) creates it offline-only; pass e.g.
+        ``online={"enabled": True, "sampleRate": 0.1, "alertThreshold": 5}`` to score live
+        traffic from the start. Section dicts use the wire's camelCase keys."""
+        payload: Dict[str, Any] = {"name": name}
+        if description is not None:
+            payload["description"] = description
+        if judge is not None:
+            payload["judge"] = judge
+        if offline is not None:
+            payload["offline"] = offline
+        if online is not None:
+            payload["online"] = online
+        return JudgeScorer(self._request("POST", "/judge-scorers", json=payload)["judgeScorer"])
+
+    def get(self, scorer_id: str) -> JudgeScorer:
+        return JudgeScorer(self._request("GET", f"/judge-scorers/{scorer_id}")["judgeScorer"])
+
+    def list(self) -> List[JudgeScorer]:
+        return [JudgeScorer(s) for s in self._request("GET", "/judge-scorers").get("judgeScorers", [])]
+
+    def update(
+        self,
+        scorer_id: str,
+        *,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        offline: Optional[Dict[str, Any]] = None,
+        online: Any = _SENTINEL,
+    ) -> JudgeScorer:
+        """Sparse update: only the sections you pass change. ``online={...}`` upserts the
+        online profile (this is how an offline-only scorer goes live); ``online=None``
+        detaches it (refused for the built-in Session Baseline Judge)."""
+        payload: Dict[str, Any] = {}
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+        if judge is not None:
+            payload["judge"] = judge
+        if offline is not None:
+            payload["offline"] = offline
+        if online is not _SENTINEL:
+            payload["online"] = online
+        return JudgeScorer(self._request("PUT", f"/judge-scorers/{scorer_id}", json=payload)["judgeScorer"])
+
+    def delete(self, scorer_id: str) -> None:
+        """Delete the scorer: rubric, version history, and online profile together.
+        Irreversible; refused for the built-in Session Baseline Judge."""
+        self._request("DELETE", f"/judge-scorers/{scorer_id}")
+
+    # ------------------------------------------------------------------
+    # Online-profile pass-throughs (calibration / tuning / ratings / events)
+    # These endpoints key on the online profile's id, resolved here so callers only ever
+    # handle the scorer id.
+    # ------------------------------------------------------------------
+
+    def _profile_id(self, scorer_id: str) -> str:
+        scorer = self.get(scorer_id)
+        profile_id = scorer.online_profile_id
+        if not profile_id:
+            raise AgentXJudgeScorersError(
+                f"Judge scorer {scorer_id!r} has no online profile - calibration/tuning/ratings "
+                "cover live-traffic scoring. Give it one first: "
+                "update(scorer_id, online={'enabled': True})."
+            )
+        return profile_id
+
+    def calibration(self, scorer_id: str, window: str = "7d") -> dict:
+        """How this scorer's verdicts compare against recorded ground truth (triage
+        corrections, outcomes, end-user votes) over the window."""
+        return self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/calibration?window={window}")
+
+    def tune(self, scorer_id: str, window: str = "7d") -> dict:
+        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow)."""
+        data = self._request(
+            "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune", json={"window": window}, timeout=300
+        )
+        # The wire wraps the proposal ({"proposal": {...}}); unwrap like the legacy client so
+        # proposal["reasoning"] / the criteria fields are directly addressable.
+        return data.get("proposal", data) if isinstance(data, dict) else data
+
+    def validate_tuning(self, scorer_id: str, criteria: Dict[str, Any], window: str = "7d") -> dict:
+        """Re-judge the disagreement + control cases with candidate criteria (LLM calls, slow)."""
+        # The wire takes the criteria fields at the TOP level of the body, not nested.
+        return self._request(
+            "POST",
+            f"/online-evaluators/{self._profile_id(scorer_id)}/tune/validate",
+            json={**criteria, "window": window},
+            timeout=600,
+        )
+
+    def publish_tuning(self, scorer_id: str, criteria: Dict[str, Any]) -> dict:
+        """Write tuned criteria onto the scorer's rubric - it applies everywhere the scorer is
+        used: online scoring, offline dataset runs, and the playground."""
+        return self._request(
+            "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune/publish", json=dict(criteria)
+        )
+
+    def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]":
+        """Bucketed average-rating-over-time for this scorer's live checks - same typed points
+        the legacy online_evaluators client returns, so scripts migrate without shape changes."""
+        from agentx.monitor.models import OnlineEvaluatorRatingPoint
+
+        data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/ratings?window={window}")
+        return [OnlineEvaluatorRatingPoint(**p) for p in data.get("points", [])]
+
+    def events(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorEvent]":
+        """Individually scored traces behind the ratings series, worst-rated first - typed, same
+        as the legacy online_evaluators client."""
+        from agentx.monitor.models import OnlineEvaluatorEvent
+
+        data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/events?window={window}")
+        return [OnlineEvaluatorEvent(**e) for e in data.get("events", [])]
+
+
+class JudgeScorerBuilder:
+    """Assembled by :meth:`JudgeScorersClient.builder`; ``.publish()`` creates the scorer."""
+
+    def __init__(
+        self,
+        client: "JudgeScorersClient",
+        *,
+        name: str,
+        description: Optional[str],
+        judge: Dict[str, Any],
+        offline: Dict[str, Any],
+        online: Optional[Dict[str, Any]],
+    ):
+        self._client = client
+        self._name = name
+        self._description = description
+        self._judge = judge
+        self._offline = offline
+        self._online = online
+
+    def publish(self) -> JudgeScorer:
+        return self._client.create(
+            self._name,
+            description=self._description,
+            judge=self._judge or None,
+            offline=self._offline or None,
+            online=self._online,
+        )