agentx-python 0.6.35__tar.gz → 0.6.36__tar.gz

{agentx_python-0.6.35 → agentx_python-0.6.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.35
+Version: 0.6.36
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/agentx.py

@@ -94,8 +94,18 @@ class AgentX:
 
     @classmethod
     def from_env(cls) -> "AgentX":
-        """Create an AgentX client using AGENTX_API_KEY (and optionally AGENTX_API_BASE_URL) from the environment."""
-        return cls()
+        """Create an AgentX client from the environment: AGENTX_API_KEY plus, for the base URL,
+        the first of AGENTX_API_BASE_URL / AGENTX_SELFHOST_BASE_URL / BASE_URL that is set.
+        The fallbacks match the conventions the self-host samples and .env files already use,
+        so from_env works wherever an explicit AgentX(base_url=...) would."""
+        import os
+
+        base_url = (
+            os.getenv("AGENTX_API_BASE_URL")
+            or os.getenv("AGENTX_SELFHOST_BASE_URL")
+            or os.getenv("BASE_URL")
+        )
+        return cls(base_url=base_url) if base_url else cls()
 
     def get_agent(self, id: str) -> Agent:
         url = f"{self.base_url or api_base()}/access/agents/{id}"

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/evaluations/client.py

@@ -59,6 +59,15 @@ class AgentXValidationError(AgentXEvaluationsError):
     pass
 
 
+def _resolve_scorer_id(scorer_id: Optional[str], evaluation_settings_id: Optional[str]) -> Optional[str]:
+    """One grader, two spellings: ``scorer_id`` is the post-consolidation name for what the wire
+    still calls ``evaluationSettingsId`` (the ids are identical by design). Both kwargs are
+    accepted everywhere a run picks its grader; passing both with different values is a bug."""
+    if scorer_id and evaluation_settings_id and scorer_id != evaluation_settings_id:
+        raise ValueError("Pass either scorer_id or evaluation_settings_id (they are the same id), not two different ids")
+    return scorer_id or evaluation_settings_id
+
+
 class EvaluationsClient:
     def __init__(
         self,
@@ -100,11 +109,21 @@ class EvaluationsClient:
         from agentx.evaluations.prompts import PromptClient
 
         self.datasets = DatasetClient(self)
-        self.settings = EvaluationSettingsClient(self)
+        # Legacy view of an LLM Judge Scorer's offline profile - constructed lazily so its
+        # DeprecationWarning fires on first USE, not for every client that never touches it.
+        self._settings: "EvaluationSettingsClient | None" = None
         self.prompts = PromptClient(self)
         from agentx.evaluations.tool_schemas import ToolSchemaClient
         self.tool_schemas = ToolSchemaClient(self)
 
+    @property
+    def settings(self) -> "EvaluationSettingsClient":
+        if self._settings is None:
+            from agentx.evaluations.evaluation_settings import EvaluationSettingsClient
+
+            self._settings = EvaluationSettingsClient(self)
+        return self._settings
+
     # ------------------------------------------------------------------
     # Low-level HTTP
     # ------------------------------------------------------------------
@@ -275,10 +294,16 @@ class EvaluationsClient:
         dataset_id: str,
         subject: EvaluationSubject,
         python_version: Optional[str] = None,
+        scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRun:
+        """``scorer_id`` names the LLM Judge Scorer grading this run (its id doubles as the
+        wire's ``evaluationSettingsId``). ``evaluation_settings_id`` is the pre-consolidation
+        alias and keeps working."""
         from agentx.version import VERSION
 
+        grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
+
         payload = {
             "datasetId": dataset_id,
             "evaluationSubject": subject.model_dump(by_alias=True, exclude_none=True),
@@ -290,8 +315,8 @@ class EvaluationsClient:
                 "pythonVersion": python_version or _python_version(),
             },
         }
-        if evaluation_settings_id:
-            payload["evaluationSettingsId"] = evaluation_settings_id
+        if grader_id:
+            payload["evaluationSettingsId"] = grader_id
         data = self._request("POST", "/runs", json=self._with_workspace(payload))
         return EvaluationRun(**data)
 

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/evaluations/datasets.py

@@ -290,6 +290,7 @@ class DatasetClient:
         rouge_score: bool = False,
         similarity_model: Optional[str] = None,
         sovereignty_models: Optional[List[str]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
     ) -> DatasetBuilder:
         return DatasetBuilder(
             self._client,
@@ -307,6 +308,9 @@ class DatasetClient:
             rouge_score=rouge_score,
             similarity_model=similarity_model,
             sovereignty_models=sovereignty_models,
+            # Was documented (evaluation/code-scorers.mdx) but not forwarded - DatasetBuilder
+            # itself always accepted it. Fixed with the judge-scorer unification.
+            code_scorers=code_scorers,
         )
 
     def from_csv(self, path: str, name: str, **kwargs) -> DatasetBuilder:

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/evaluations/evaluation_settings.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
 from agentx.evaluations.models import EvaluationSettings
@@ -81,10 +82,25 @@ class EvaluationSettingsBuilder:
 
 
 class EvaluationSettingsClient:
-    """Thin wrapper surfaced as client.evaluations.settings."""
+    """Thin wrapper surfaced as ``client.evaluations.settings``.
+
+    Note: an evaluation-settings record is the judge rubric + OFFLINE profile of an
+    **LLM Judge Scorer** - the unified entity at ``client.monitor.judge_scorers``, which also
+    carries the optional online (live-traffic) profile. This client keeps working unchanged;
+    prefer ``judge_scorers`` for new code so both profiles live in one place -
+    ``client.monitor.judge_scorers.builder(...)`` has the same snake_case ergonomics as the
+    builder below, plus tool_context, thresholds, and the live profile in one call."""
 
     def __init__(self, client: "EvaluationsClient"):
         self._client = client
+        # Soft deprecation: hidden by default (DeprecationWarning), visible under -W or pytest.
+        warnings.warn(
+            "client.evaluations.settings is the legacy view of an LLM Judge Scorer's offline "
+            "profile; prefer client.monitor.judge_scorers, which manages the judge rubric, "
+            "offline profile, and online (live-traffic) profile as one entity.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
 
     def builder(
         self,

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/evaluations/models.py

@@ -94,7 +94,7 @@ class Dataset(BaseModel):
 
 class EvaluationSettings(BaseModel):
     """A standalone, reusable grading config - no dataset/questions attached.
-    Created via ``client.evaluations.settings.builder(...).publish()`` and run
+    Created via ``client.monitor.judge_scorers.builder(...).publish()`` (or the legacy ``client.evaluations.settings.builder``) and run
     against any dataset by passing its id as ``evaluation_settings_id`` to
     ``client.evaluations.run(...)``."""
 

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/evaluations/runner.py

@@ -421,10 +421,15 @@ class EvaluationsRunner:
     def __init__(self, client: EvaluationsClient):
         self._client = client
         self.datasets = client.datasets
-        self.settings = client.settings
         self.prompts = client.prompts
         self.tool_schemas = client.tool_schemas
 
+    @property
+    def settings(self):
+        # Deferred: touching client.settings eagerly here would fire its legacy-surface
+        # DeprecationWarning for every runner, used or not.
+        return self._client.settings
+
     def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
         """List the LLM models AgentX supports - the same set selectable for
         the Sovereignty & Portability Index. Pass ``provider`` (e.g. "Google")
@@ -482,24 +487,24 @@ class EvaluationsRunner:
         self,
         dataset_id: str,
         subject: Union[Dict[str, Any], EvaluationSubject],
+        scorer_id: Optional[str] = None,
         evaluation_settings_id: Optional[str] = None,
     ) -> EvaluationRunContext:
-        """Start a run of ``dataset_id`` against ``subject``. Pass
-        ``evaluation_settings_id`` to grade against a standalone, reusable
-        config (created via ``client.evaluations.settings.builder(...)``)
-        instead of the dataset's own default config."""
+        """Start a run of ``dataset_id`` against ``subject``. Pass ``scorer_id`` (an LLM Judge
+        Scorer's id, e.g. from ``client.monitor.judge_scorers``) to grade with a specific
+        scorer instead of the dataset's default. ``evaluation_settings_id`` is the
+        pre-consolidation alias for the same id and keeps working."""
+        from agentx.evaluations.client import _resolve_scorer_id
+
         if isinstance(subject, dict):
             subject = EvaluationSubject(**subject)
 
+        grader_id = _resolve_scorer_id(scorer_id, evaluation_settings_id)
         dataset = self._client.get_dataset(dataset_id)
         evaluation_settings = (
-            self._client.get_evaluation_settings(evaluation_settings_id)
-            if evaluation_settings_id
-            else None
-        )
-        run = self._client.init_run(
-            dataset_id, subject, evaluation_settings_id=evaluation_settings_id
+            self._client.get_evaluation_settings(grader_id) if grader_id else None
         )
+        run = self._client.init_run(dataset_id, subject, scorer_id=grader_id)
         logger.info(
             "Started evaluation run %s on dataset %s (%d case(s), %d repetition(s))",
             run.run_id,

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/monitor/client.py

@@ -88,8 +88,15 @@ class MonitorClient:
         # Handed this client's own resolved API root, never the process-global default, so a
         # second AgentX() with a different base_url can't re-point it (deep-dive bug #1).
         self.scorers = ScorersClient(api_key=api_key, base_url=self._api_root())
+        from agentx.monitor.judge_scorers import JudgeScorersClient
+        # The unified LLM Judge Scorer (rubric + offline/online profiles in one entity) - the
+        # surface that matches the product; evaluations.settings and online_evaluators below
+        # remain as its profile-level views.
+        self.judge_scorers = JudgeScorersClient(api_key=api_key, base_url=self._api_root())
         self.profile = MonitorProfileClient(self)
-        self.online_evaluators = MonitorOnlineEvaluatorClient(self)
+        # Legacy view of an LLM Judge Scorer's online profile - constructed lazily so its
+        # DeprecationWarning fires on first USE, not for every client that never touches it.
+        self._online_evaluators: "MonitorOnlineEvaluatorClient | None" = None
         from agentx.monitor.sessions import MonitorSessionClient
         from agentx.monitor.agents import MonitorAgentClient
         self.sessions = MonitorSessionClient(self)
@@ -173,6 +180,14 @@ class MonitorClient:
         data = self._request("POST", "/online-evaluators", json=self._with_workspace(payload))
         return MonitorOnlineEvaluator(**data["evaluator"])
 
+    @property
+    def online_evaluators(self) -> "MonitorOnlineEvaluatorClient":
+        if self._online_evaluators is None:
+            from agentx.monitor.online_evaluators import MonitorOnlineEvaluatorClient
+
+            self._online_evaluators = MonitorOnlineEvaluatorClient(self)
+        return self._online_evaluators
+
     def list_online_evaluators(self) -> List[MonitorOnlineEvaluator]:
         data = self._request("GET", "/online-evaluators", params=self._workspace_params())
         return [MonitorOnlineEvaluator(**e) for e in data.get("evaluators", [])]

agentx_python-0.6.36/agentx/monitor/judge_scorers.py

@@ -0,0 +1,334 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from agentx.util import api_base, get_headers
+
+logger = logging.getLogger(__name__)
+
+_SENTINEL: Any = object()
+
+
+class AgentXJudgeScorersError(Exception):
+    pass
+
+
+class JudgeScorer(dict):
+    """Wire object for the unified LLM Judge Scorer (dict subclass so unknown fields
+    round-trip). Convenience properties expose the pieces scripts reach for most."""
+
+    @property
+    def id(self) -> str:
+        return self["_id"]
+
+    @property
+    def name(self) -> str:
+        return self["name"]
+
+    @property
+    def judge(self) -> Dict[str, Any]:
+        return self.get("judge", {})
+
+    @property
+    def offline(self) -> Dict[str, Any]:
+        return self.get("offline", {})
+
+    @property
+    def online(self) -> Optional[Dict[str, Any]]:
+        return self.get("online")
+
+    @property
+    def online_profile_id(self) -> Optional[str]:
+        online = self.online
+        return online.get("profileId") if online else None
+
+
+class JudgeScorersClient:
+    """Surfaced as ``client.monitor.judge_scorers``: the unified LLM Judge Scorer.
+
+    One scorer = one judge rubric (``judge``: acceptance/rejection/evaluation criteria, judge
+    prompt, judge model) + two setting profiles:
+
+    - ``offline`` - how dataset runs grade with it (repetitions, similarity metrics, code
+      scorers, default/status). The scorer's ``id`` is exactly what
+      ``client.evaluations.run(dataset_id, scorer_id=scorer.id)`` takes.
+    - ``online`` - whether/how it scores live traffic (enabled, sample rate, scope,
+      alert threshold). ``None`` means offline-only.
+
+    Strictly one online profile per scorer. This supersedes the split between
+    ``client.evaluations.settings`` (the offline half) and ``client.monitor.online_evaluators``
+    (the online half) - both keep working, but this is the surface that matches the product.
+
+    Example::
+
+        scorer = client.monitor.judge_scorers.create(
+            "Support quality",
+            judge={"acceptanceCriteria": "Concrete, correct, cites the policy."},
+            online={"enabled": True, "sampleRate": 0.2, "alertThreshold": 6},
+        )
+        client.evaluations.run(dataset_id, subject, scorer_id=scorer.id)
+        cal = client.monitor.judge_scorers.calibration(scorer.id)
+    """
+
+    def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None):
+        self._api_key = api_key
+        # Captured once at construction so two clients with different bases can coexist.
+        self._base_url = (base_url or api_base()).rstrip("/")
+
+    def _request(self, method: str, path: str, json: Any = None, timeout: int = 60) -> Any:
+        resp = requests.request(
+            method,
+            f"{self._base_url}/agent-monitoring{path}",
+            headers={**get_headers(self._api_key), "Content-Type": "application/json"},
+            json=json,
+            timeout=timeout,
+        )
+        if resp.status_code >= 400:
+            try:
+                detail = resp.json().get("error", resp.reason)
+            except ValueError:
+                detail = resp.reason
+            raise AgentXJudgeScorersError(f"Judge scorer request failed ({resp.status_code}): {detail}")
+        return resp.json() if resp.text else {}
+
+    # ------------------------------------------------------------------
+    # CRUD
+    # ------------------------------------------------------------------
+
+    def builder(
+        self,
+        name: str,
+        description: Optional[str] = None,
+        *,
+        # Judge rubric (shared by every surface that runs this scorer)
+        acceptance_criteria: Optional[str] = None,
+        rejection_criteria: Optional[str] = None,
+        evaluation_criteria: Optional[str] = None,
+        judge_prompt: Optional[str] = None,
+        judge_model: Optional[str] = None,
+        tool_context: Optional[str] = None,
+        # Offline profile (dataset-run grading)
+        number_of_requests: int = 1,
+        vector_similarity: bool = False,
+        jaccard_similarity: bool = False,
+        bleu_score: bool = False,
+        rouge_score: bool = False,
+        similarity_model: Optional[str] = None,
+        sovereignty_models: Optional[List[str]] = None,
+        thresholds: Optional[Dict[str, Any]] = None,
+        code_scorers: Optional[List[Dict[str, Any]]] = None,
+        is_default: bool = False,
+        # Online profile (live traffic). live=True enables scoring immediately - every check is
+        # a real judge call on your own key.
+        live: bool = False,
+        sample_rate: float = 0.1,
+        scope: str = "trace",
+        alert_threshold: Optional[float] = 5,
+        severity: str = "medium",
+        agent_ids: Optional[List[str]] = None,
+        idle_seconds: int = 120,
+    ) -> "JudgeScorerBuilder":
+        """Snake_case builder with ``.publish()``, the unified successor of
+        ``client.evaluations.settings.builder(...)`` - same offline fields (plus ``thresholds``,
+        ``tool_context``) and, new here, the online profile in the same call. The scorer the
+        builder publishes is one entity: its ``.id`` is what ``client.evaluations.run(...,
+        scorer_id=...)`` takes, and its live profile is what online scoring keys on."""
+        judge: Dict[str, Any] = {}
+        for key, value in (
+            ("acceptanceCriteria", acceptance_criteria),
+            ("rejectionCriteria", rejection_criteria),
+            ("evaluationCriteria", evaluation_criteria),
+            ("judgePrompt", judge_prompt),
+            ("judgeModel", judge_model),
+            ("toolContext", tool_context),
+        ):
+            if value is not None:
+                judge[key] = value
+        offline: Dict[str, Any] = {"numberOfRequests": number_of_requests, "isDefault": is_default}
+        if vector_similarity:
+            offline["vectorSimilarity"] = (
+                {"enabled": True, "model": similarity_model} if similarity_model else {"enabled": True}
+            )
+        if jaccard_similarity:
+            offline["jaccardSimilarity"] = {"enabled": True}
+        if bleu_score:
+            offline["bleuScore"] = {"enabled": True}
+        if rouge_score:
+            offline["rougeScore"] = {"enabled": True}
+        if sovereignty_models is not None:
+            offline["sovereigntyIndex"] = {"enabled": bool(sovereignty_models), "models": sovereignty_models}
+        if thresholds is not None:
+            offline["thresholds"] = thresholds
+        if code_scorers is not None:
+            offline["codeScorers"] = code_scorers
+        online: Optional[Dict[str, Any]] = None
+        if live:
+            online = {
+                "enabled": True,
+                "sampleRate": sample_rate,
+                "scope": scope,
+                "alertThreshold": alert_threshold,
+                "severity": severity,
+                "idleSeconds": idle_seconds,
+            }
+            if agent_ids:
+                online["scopeMode"] = "selected"
+                online["agentIds"] = agent_ids
+        return JudgeScorerBuilder(self, name=name, description=description, judge=judge, offline=offline, online=online)
+
+    def create(
+        self,
+        name: str,
+        *,
+        description: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        offline: Optional[Dict[str, Any]] = None,
+        online: Optional[Dict[str, Any]] = None,
+    ) -> JudgeScorer:
+        """Create a scorer. ``online=None`` (default) creates it offline-only; pass e.g.
+        ``online={"enabled": True, "sampleRate": 0.1, "alertThreshold": 5}`` to score live
+        traffic from the start. Section dicts use the wire's camelCase keys."""
+        payload: Dict[str, Any] = {"name": name}
+        if description is not None:
+            payload["description"] = description
+        if judge is not None:
+            payload["judge"] = judge
+        if offline is not None:
+            payload["offline"] = offline
+        if online is not None:
+            payload["online"] = online
+        return JudgeScorer(self._request("POST", "/judge-scorers", json=payload)["judgeScorer"])
+
+    def get(self, scorer_id: str) -> JudgeScorer:
+        return JudgeScorer(self._request("GET", f"/judge-scorers/{scorer_id}")["judgeScorer"])
+
+    def list(self) -> List[JudgeScorer]:
+        return [JudgeScorer(s) for s in self._request("GET", "/judge-scorers").get("judgeScorers", [])]
+
+    def update(
+        self,
+        scorer_id: str,
+        *,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        judge: Optional[Dict[str, Any]] = None,
+        offline: Optional[Dict[str, Any]] = None,
+        online: Any = _SENTINEL,
+    ) -> JudgeScorer:
+        """Sparse update: only the sections you pass change. ``online={...}`` upserts the
+        online profile (this is how an offline-only scorer goes live); ``online=None``
+        detaches it (refused for the built-in Session Baseline Judge)."""
+        payload: Dict[str, Any] = {}
+        if name is not None:
+            payload["name"] = name
+        if description is not None:
+            payload["description"] = description
+        if judge is not None:
+            payload["judge"] = judge
+        if offline is not None:
+            payload["offline"] = offline
+        if online is not _SENTINEL:
+            payload["online"] = online
+        return JudgeScorer(self._request("PUT", f"/judge-scorers/{scorer_id}", json=payload)["judgeScorer"])
+
+    def delete(self, scorer_id: str) -> None:
+        """Delete the scorer: rubric, version history, and online profile together.
+        Irreversible; refused for the built-in Session Baseline Judge."""
+        self._request("DELETE", f"/judge-scorers/{scorer_id}")
+
+    # ------------------------------------------------------------------
+    # Online-profile pass-throughs (calibration / tuning / ratings / events)
+    # These endpoints key on the online profile's id, resolved here so callers only ever
+    # handle the scorer id.
+    # ------------------------------------------------------------------
+
+    def _profile_id(self, scorer_id: str) -> str:
+        scorer = self.get(scorer_id)
+        profile_id = scorer.online_profile_id
+        if not profile_id:
+            raise AgentXJudgeScorersError(
+                f"Judge scorer {scorer_id!r} has no online profile - calibration/tuning/ratings "
+                "cover live-traffic scoring. Give it one first: "
+                "update(scorer_id, online={'enabled': True})."
+            )
+        return profile_id
+
+    def calibration(self, scorer_id: str, window: str = "7d") -> dict:
+        """How this scorer's verdicts compare against recorded ground truth (triage
+        corrections, outcomes, end-user votes) over the window."""
+        return self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/calibration?window={window}")
+
+    def tune(self, scorer_id: str, window: str = "7d") -> dict:
+        """Propose a rewrite of the rubric from calibration disagreements (LLM call, slow)."""
+        data = self._request(
+            "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune", json={"window": window}, timeout=300
+        )
+        # The wire wraps the proposal ({"proposal": {...}}); unwrap like the legacy client so
+        # proposal["reasoning"] / the criteria fields are directly addressable.
+        return data.get("proposal", data) if isinstance(data, dict) else data
+
+    def validate_tuning(self, scorer_id: str, criteria: Dict[str, Any], window: str = "7d") -> dict:
+        """Re-judge the disagreement + control cases with candidate criteria (LLM calls, slow)."""
+        # The wire takes the criteria fields at the TOP level of the body, not nested.
+        return self._request(
+            "POST",
+            f"/online-evaluators/{self._profile_id(scorer_id)}/tune/validate",
+            json={**criteria, "window": window},
+            timeout=600,
+        )
+
+    def publish_tuning(self, scorer_id: str, criteria: Dict[str, Any]) -> dict:
+        """Write tuned criteria onto the scorer's rubric - it applies everywhere the scorer is
+        used: online scoring, offline dataset runs, and the playground."""
+        return self._request(
+            "POST", f"/online-evaluators/{self._profile_id(scorer_id)}/tune/publish", json=dict(criteria)
+        )
+
+    def ratings(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorRatingPoint]":
+        """Bucketed average-rating-over-time for this scorer's live checks - same typed points
+        the legacy online_evaluators client returns, so scripts migrate without shape changes."""
+        from agentx.monitor.models import OnlineEvaluatorRatingPoint
+
+        data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/ratings?window={window}")
+        return [OnlineEvaluatorRatingPoint(**p) for p in data.get("points", [])]
+
+    def events(self, scorer_id: str, window: str = "7d") -> "List[OnlineEvaluatorEvent]":
+        """Individually scored traces behind the ratings series, worst-rated first - typed, same
+        as the legacy online_evaluators client."""
+        from agentx.monitor.models import OnlineEvaluatorEvent
+
+        data = self._request("GET", f"/online-evaluators/{self._profile_id(scorer_id)}/events?window={window}")
+        return [OnlineEvaluatorEvent(**e) for e in data.get("events", [])]
+
+
+class JudgeScorerBuilder:
+    """Assembled by :meth:`JudgeScorersClient.builder`; ``.publish()`` creates the scorer."""
+
+    def __init__(
+        self,
+        client: "JudgeScorersClient",
+        *,
+        name: str,
+        description: Optional[str],
+        judge: Dict[str, Any],
+        offline: Dict[str, Any],
+        online: Optional[Dict[str, Any]],
+    ):
+        self._client = client
+        self._name = name
+        self._description = description
+        self._judge = judge
+        self._offline = offline
+        self._online = online
+
+    def publish(self) -> JudgeScorer:
+        return self._client.create(
+            self._name,
+            description=self._description,
+            judge=self._judge or None,
+            offline=self._offline or None,
+            online=self._online,
+        )

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx/monitor/online_evaluators.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import Any, Dict, List, Optional, TYPE_CHECKING
 
 from agentx.monitor.models import MonitorOnlineEvaluator, OnlineEvaluatorRatingPoint, OnlineEvaluatorEvent
@@ -17,7 +18,12 @@ class MonitorOnlineEvaluatorBuilder:
     ``evaluation_settings_id`` must reference an existing Evaluator config (criteria, judge
     prompt, judge model), the same config datasets/Evaluate runs use, see
     ``client.evaluations.settings.builder(...)``.
-    """
+
+    Note: an online evaluator is the ONLINE profile of an **LLM Judge Scorer** - the unified
+    entity at ``client.monitor.judge_scorers``. Strictly one profile per config since the
+    unification: binding a config that is already another evaluator's profile transparently
+    binds a fresh copy of it instead (the response's ``evaluationSettingsId`` is the copy).
+    This surface keeps working unchanged; prefer ``judge_scorers`` for new code."""
 
     def __init__(
         self,
@@ -60,10 +66,23 @@ class MonitorOnlineEvaluatorBuilder:
 
 
 class MonitorOnlineEvaluatorClient:
-    """Thin wrapper surfaced as ``client.monitor.online_evaluators``."""
+    """Thin wrapper surfaced as ``client.monitor.online_evaluators``.
+
+    Note: an online evaluator is the ONLINE (live-traffic) profile of an **LLM Judge Scorer** -
+    the unified entity at ``client.monitor.judge_scorers``, which also carries the judge rubric
+    and the offline profile. This client keeps working unchanged; prefer ``judge_scorers`` for
+    new code so both profiles live in one place."""
 
     def __init__(self, client: "MonitorClient"):
         self._client = client
+        # Soft deprecation: hidden by default (DeprecationWarning), visible under -W or pytest.
+        warnings.warn(
+            "client.monitor.online_evaluators is the legacy view of an LLM Judge Scorer's "
+            "online profile; prefer client.monitor.judge_scorers, which manages the judge "
+            "rubric, offline profile, and online profile as one entity.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
 
     def builder(
         self,

agentx_python-0.6.36/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.36"

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.35
+Version: 0.6.36
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team

{agentx_python-0.6.35 → agentx_python-0.6.36}/agentx_python.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ agentx/integrations/openai_agents.py
 agentx/monitor/__init__.py
 agentx/monitor/agents.py
 agentx/monitor/client.py
+agentx/monitor/judge_scorers.py
 agentx/monitor/models.py
 agentx/monitor/online_evaluators.py
 agentx/monitor/patterns.py
@@ -71,5 +72,6 @@ agentx_python.egg-info/top_level.txt
 tests/test_deep_dive_fixes.py
 tests/test_integration.py
 tests/test_integrations.py
+tests/test_judge_scorers.py
 tests/test_selfhost_analysis_fallback.py
 tests/test_span_tree.py

agentx_python-0.6.36/tests/test_judge_scorers.py

@@ -0,0 +1,270 @@
+"""Unit tests for the unified LLM Judge Scorer surface (client.monitor.judge_scorers) and the
+dataset-builder code_scorers forwarding fix - wire-level, no engine required (requests is
+monkeypatched; the engine-side contract is pinned by the engine's own
+judgeScorers.integration.test.ts)."""
+
+from typing import Any, Dict, List, Optional
+
+import pytest
+
+from agentx.monitor.judge_scorers import AgentXJudgeScorersError, JudgeScorersClient
+
+
+class FakeResponse:
+    def __init__(self, payload: Dict[str, Any], status_code: int = 200):
+        self._payload = payload
+        self.status_code = status_code
+        self.text = "x"
+        self.reason = "reason"
+
+    def json(self) -> Dict[str, Any]:
+        return self._payload
+
+
+@pytest.fixture()
+def recorded(monkeypatch):
+    calls: List[Dict[str, Any]] = []
+    responses: List[FakeResponse] = []
+
+    def fake_request(method, url, headers=None, json=None, timeout=None):
+        calls.append({"method": method, "url": url, "json": json})
+        return responses.pop(0) if responses else FakeResponse({"judgeScorer": {"_id": "s1", "name": "n"}})
+
+    monkeypatch.setattr("agentx.monitor.judge_scorers.requests.request", fake_request)
+    return calls, responses
+
+
+def make_client() -> JudgeScorersClient:
+    return JudgeScorersClient(api_key="k", base_url="http://engine:4700/api/v1")
+
+
+def test_create_hits_the_unified_route_with_sections(recorded):
+    calls, _ = recorded
+    make_client().create(
+        "Support quality",
+        judge={"acceptanceCriteria": "Concrete."},
+        online={"enabled": True, "sampleRate": 0.2},
+    )
+    assert calls[0]["method"] == "POST"
+    assert calls[0]["url"].endswith("/agent-monitoring/judge-scorers")
+    assert calls[0]["json"] == {
+        "name": "Support quality",
+        "judge": {"acceptanceCriteria": "Concrete."},
+        "online": {"enabled": True, "sampleRate": 0.2},
+    }
+
+
+def test_update_distinguishes_online_none_from_online_absent(recorded):
+    calls, _ = recorded
+    client = make_client()
+    client.update("s1", judge={"judgePrompt": "p"})
+    assert "online" not in calls[0]["json"]  # absent section untouched
+
+    client.update("s1", online=None)
+    assert calls[1]["json"] == {"online": None}  # explicit detach
+
+
+def test_online_passthroughs_resolve_the_profile_id(recorded):
+    calls, responses = recorded
+    responses.append(FakeResponse({"judgeScorer": {"_id": "s1", "name": "n", "online": {"profileId": "prof-9"}}}))
+    responses.append(FakeResponse({"agreementRate": 0.5}))
+    result = make_client().calibration("s1", window="24h")
+    assert result == {"agreementRate": 0.5}
+    assert calls[1]["url"].endswith("/agent-monitoring/online-evaluators/prof-9/calibration?window=24h")
+
+
+def test_offline_only_scorers_get_a_clear_error_for_online_surfaces(recorded):
+    _, responses = recorded
+    responses.append(FakeResponse({"judgeScorer": {"_id": "s1", "name": "n", "online": None}}))
+    with pytest.raises(AgentXJudgeScorersError, match="no online profile"):
+        make_client().ratings("s1")
+
+
+def test_http_errors_surface_the_engine_message(recorded):
+    _, responses = recorded
+    responses.append(FakeResponse({"error": "built-in judge scorer"}, status_code=409))
+    with pytest.raises(AgentXJudgeScorersError, match="409.*built-in judge scorer"):
+        make_client().delete("baseline")
+
+
+def test_dataset_builder_forwards_code_scorers(monkeypatch):
+    # The documented kwarg (evaluation/code-scorers.mdx) raised TypeError before the fix.
+    from agentx.evaluations.datasets import DatasetClient
+
+    class FakeEvalClient:
+        def create_dataset(self, payload):
+            self.payload = payload
+            return {"_id": "d1"}
+
+    fake = FakeEvalClient()
+    builder = DatasetClient(fake).builder(
+        "ds", code_scorers=[{"name": "tool order", "code": "def handler(...): pass"}]
+    )
+    payload = builder._payload  # noqa: SLF001 - asserting the wire body the builder assembled
+    assert "codeScorers" in payload
+    assert payload["codeScorers"][0]["name"] == "tool order"
+    assert payload["codeScorers"][0]["enabled"] is True
+    assert payload["codeScorers"][0]["id"]
+
+
+def test_legacy_profile_clients_warn_once_on_first_access():
+    """The legacy views (evaluations.settings / monitor.online_evaluators) stay functional but
+    emit a DeprecationWarning pointing at judge_scorers - lazily, so a client that never touches
+    them never warns."""
+    import warnings
+
+    from agentx import AgentX
+
+    with warnings.catch_warnings(record=True) as during_init:
+        warnings.simplefilter("always")
+        client = AgentX(api_key="agtx_local_test", base_url="http://localhost:1")
+        _ = client.monitor.judge_scorers  # the unified surface never warns
+    assert not [w for w in during_init if "judge_scorers" in str(w.message)]
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        _ = client.evaluations.settings
+        _ = client.monitor.online_evaluators
+    messages = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)]
+    assert any("judge_scorers" in m for m in messages), messages
+    assert len(messages) >= 2
+
+
+def test_scorer_id_is_the_preferred_run_kwarg():
+    """Runs pick their grader as scorer_id (post-consolidation name); the legacy
+    evaluation_settings_id kwarg maps to the same wire field, and passing two DIFFERENT ids is
+    rejected."""
+    import pytest
+
+    from agentx.evaluations.client import _resolve_scorer_id
+
+    assert _resolve_scorer_id("s1", None) == "s1"
+    assert _resolve_scorer_id(None, "s1") == "s1"
+    assert _resolve_scorer_id("s1", "s1") == "s1"
+    assert _resolve_scorer_id(None, None) is None
+    with pytest.raises(ValueError):
+        _resolve_scorer_id("s1", "s2")
+
+
+def test_init_run_sends_scorer_id_as_evaluationSettingsId(monkeypatch):
+    from agentx.evaluations.client import EvaluationsClient
+    from agentx.evaluations.models import EvaluationSubject
+
+    client = EvaluationsClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    captured = {}
+
+    def fake_request(method, path, **kwargs):
+        captured["payload"] = kwargs.get("json")
+        return {
+            "runId": "r1",
+            "datasetId": "d1",
+            "status": "running",
+            "numberOfRequests": 1,
+        }
+
+    monkeypatch.setattr(client, "_request", fake_request)
+    client.init_run("d1", EvaluationSubject(type="external"), scorer_id="scorer-123")
+    assert captured["payload"]["evaluationSettingsId"] == "scorer-123"
+
+
+def test_judge_scorers_builder_matches_legacy_builder_ergonomics(monkeypatch):
+    """The unified successor of evaluations.settings.builder: snake_case kwargs, .publish(),
+    plus what the legacy builder never had - tool_context, thresholds, and the live profile in
+    the same call. The payload it assembles is plain judge-scorers wire."""
+    from agentx.monitor.judge_scorers import JudgeScorersClient
+
+    client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    captured = {}
+
+    def fake_request(method, path, **kwargs):
+        captured["method"] = method
+        captured["path"] = path
+        captured["payload"] = kwargs.get("json")
+        return {"judgeScorer": {"_id": "s1", "name": "Support quality", "judge": {}, "offline": {}, "online": None}}
+
+    monkeypatch.setattr(client, "_request", fake_request)
+    scorer = (
+        client.builder(
+            "Support quality",
+            acceptance_criteria="Concrete and correct.",
+            judge_model="gpt-4.1-mini",
+            tool_context="detailed",
+            number_of_requests=2,
+            vector_similarity=True,
+            thresholds={"enabled": True, "gates": [{"metric": "rating", "operator": "lt", "value": 5}]},
+            live=True,
+            sample_rate=0.25,
+            agent_ids=["support-agent"],
+        ).publish()
+    )
+    assert scorer.id == "s1"
+    payload = captured["payload"]
+    assert captured["path"] == "/judge-scorers"
+    assert payload["judge"] == {
+        "acceptanceCriteria": "Concrete and correct.",
+        "judgeModel": "gpt-4.1-mini",
+        "toolContext": "detailed",
+    }
+    assert payload["offline"]["numberOfRequests"] == 2
+    assert payload["offline"]["vectorSimilarity"] == {"enabled": True}
+    assert payload["offline"]["thresholds"]["gates"][0]["value"] == 5
+    assert payload["online"]["enabled"] is True
+    assert payload["online"]["sampleRate"] == 0.25
+    assert payload["online"]["scopeMode"] == "selected"
+    assert payload["online"]["agentIds"] == ["support-agent"]
+
+
+def test_from_env_honors_selfhost_base_url_conventions(monkeypatch):
+    """from_env silently targeting the hosted default while the shell exports the self-host
+    conventions (AGENTX_SELFHOST_BASE_URL / BASE_URL) produced confusing auth errors - it now
+    picks up the first convention that is set."""
+    from agentx import AgentX
+
+    monkeypatch.setenv("AGENTX_API_KEY", "agtx_local_test")
+    monkeypatch.delenv("AGENTX_API_BASE_URL", raising=False)
+    monkeypatch.setenv("AGENTX_SELFHOST_BASE_URL", "http://localhost:4999/api/v1")
+    client = AgentX.from_env()
+    assert client.base_url == "http://localhost:4999/api/v1"
+
+    monkeypatch.setenv("AGENTX_API_BASE_URL", "http://localhost:5000/api/v1")
+    assert AgentX.from_env().base_url == "http://localhost:5000/api/v1"  # explicit name wins
+
+
+def test_tune_unwraps_the_proposal_envelope(monkeypatch):
+    """The tune wire wraps its result in {"proposal": {...}} - judge_scorers.tune must unwrap it
+    like the legacy client does, so proposal["reasoning"]/criteria are directly addressable
+    (selfhost_demo/11 crashed on the wrapped form)."""
+    from agentx.monitor.judge_scorers import JudgeScorersClient
+
+    client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    monkeypatch.setattr(client, "_profile_id", lambda scorer_id: "prof-1")
+    monkeypatch.setattr(
+        client,
+        "_request",
+        lambda *a, **k: {"proposal": {"reasoning": "why", "acceptanceCriteria": "a"}},
+    )
+    proposal = client.tune("s1")
+    assert proposal["reasoning"] == "why"
+
+
+def test_validate_and_publish_send_criteria_at_top_level(monkeypatch):
+    """The tuning wire takes the criteria fields at the TOP level of the body (the legacy client
+    always did) - nesting them under "criteria" 400s with 'acceptanceCriteria is required'."""
+    from agentx.monitor.judge_scorers import JudgeScorersClient
+
+    client = JudgeScorersClient(api_key="agtx_local_test", base_url="http://localhost:1")
+    monkeypatch.setattr(client, "_profile_id", lambda scorer_id: "prof-1")
+    captured = {}
+
+    def fake_request(method, path, json=None, timeout=60):
+        captured[path.rsplit("/", 1)[-1]] = json
+        return {}
+
+    monkeypatch.setattr(client, "_request", fake_request)
+    criteria = {"acceptanceCriteria": "a", "rejectionCriteria": "r", "evaluationCriteria": "e"}
+    client.validate_tuning("s1", criteria, window="24h")
+    client.publish_tuning("s1", criteria)
+    assert captured["validate"]["acceptanceCriteria"] == "a"
+    assert captured["validate"]["window"] == "24h"
+    assert "criteria" not in captured["validate"]
+    assert captured["publish"]["acceptanceCriteria"] == "a"

agentx_python-0.6.35/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.35"