agentx-python 0.6.11__tar.gz → 0.6.12__tar.gz

{agentx_python-0.6.11/agentx_python.egg-info → agentx_python-0.6.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.11
+Version: 0.6.12
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -298,6 +298,19 @@ Per-agent coverage/threshold settings (sample rate, retention, and threshold ove
 client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
 ```
 
+Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
+
+```python
+evaluator = client.monitor.online_evaluators.builder(
+    name="Helpfulness",
+    evaluation_settings_id=settings.id,
+    sample_rate=0.1,
+    alert_threshold=5,
+).publish()
+
+client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -330,7 +343,7 @@ Ask a case's question several extra ways each run, LLM-paraphrased server-side,
 Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
 
 ```python
-prompt = client.evaluations.prompts.get("support-agent-system-prompt")
+prompt = client.evaluations.prompts.get("support-agent-system-prompt")  # or prompt.id
 # use prompt.text as your own agent's system prompt
 
 client.evaluations.run(

{agentx_python-0.6.11 → agentx_python-0.6.12}/README.md

@@ -236,6 +236,19 @@ Per-agent coverage/threshold settings (sample rate, retention, and threshold ove
 client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
 ```
 
+Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
+
+```python
+evaluator = client.monitor.online_evaluators.builder(
+    name="Helpfulness",
+    evaluation_settings_id=settings.id,
+    sample_rate=0.1,
+    alert_threshold=5,
+).publish()
+
+client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -268,7 +281,7 @@ Ask a case's question several extra ways each run, LLM-paraphrased server-side,
 Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
 
 ```python
-prompt = client.evaluations.prompts.get("support-agent-system-prompt")
+prompt = client.evaluations.prompts.get("support-agent-system-prompt")  # or prompt.id
 # use prompt.text as your own agent's system prompt
 
 client.evaluations.run(

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx/evaluations/client.py

@@ -213,6 +213,8 @@ class EvaluationsClient:
         return [Prompt(**p) for p in (data if isinstance(data, list) else data.get("prompts", []))]
 
     def get_prompt(self, name: str, version: Optional[int] = None) -> Prompt:
+        # `name` doubles as an id: the backend route tries a name match first, then falls back to
+        # an id match (see engine's getPromptRowByNameOrId), so callers can pass either.
         params = self._workspace_params() or {}
         if version is not None:
             params = {**params, "version": version}

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx/evaluations/models.py

@@ -132,7 +132,7 @@ class EvaluationSettings(BaseModel):
 # Prompt registry — the external-agent analog to AgentX's native autotune. AgentX doesn't own
 # your agent's code, so instead of branching/applying a config it becomes the prompt's source of
 # truth (same shape as LangSmith's Prompt Hub / Langfuse's Prompt Management): pull a version at
-# runtime with ``client.evaluations.prompts.get(name)``, use ``prompt.text`` as your agent's
+# runtime with ``client.evaluations.prompts.get(name_or_id)``, use ``prompt.text`` as your agent's
 # system prompt, and tag your eval runs so the existing version-comparison view can tell you
 # which published version scored higher — see ``client.evaluations.prompts`` docs.
 # ---------------------------------------------------------------------------

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx/evaluations/prompts.py

@@ -34,6 +34,9 @@ class PromptClient:
                 }
             ),
         )
+
+    ``get()`` also accepts a prompt's ``id`` in place of its name, e.g.
+    ``client.evaluations.prompts.get(prompt.id)``.
     """
 
     def __init__(self, client: "EvaluationsClient"):
@@ -43,6 +46,9 @@ class PromptClient:
         return self._client.create_prompt({"name": name, "text": text, "description": description})
 
     def get(self, name: str, version: Optional[int] = None) -> Prompt:
+        """Accepts either the prompt's name or its ``id`` (e.g. ``prompt.id`` from an earlier
+        ``get``/``create`` call), useful for round-tripping an id you already have without a
+        second lookup method."""
         return self._client.get_prompt(name, version=version)
 
     def list(self) -> List[Prompt]:

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx/monitor/client.py

@@ -7,7 +7,14 @@ from typing import Any, List, Optional
 
 import requests
 
-from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal
+from agentx.monitor.models import (
+    MonitorPattern,
+    MonitorProfile,
+    MonitorSignal,
+    MonitorOnlineEvaluator,
+    OnlineEvaluatorRatingPoint,
+    OnlineEvaluatorEvent,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -71,10 +78,12 @@ class MonitorClient:
         from agentx.monitor.patterns import MonitorPatternClient
         from agentx.monitor.signals import MonitorSignalClient
         from agentx.monitor.profile import MonitorProfileClient
+        from agentx.monitor.online_evaluators import MonitorOnlineEvaluatorClient
 
         self.patterns = MonitorPatternClient(self)
         self.signals = MonitorSignalClient(self)
         self.profile = MonitorProfileClient(self)
+        self.online_evaluators = MonitorOnlineEvaluatorClient(self)
 
     # ------------------------------------------------------------------
     # Low-level HTTP
@@ -137,6 +146,45 @@ class MonitorClient:
         )
         return MonitorPattern(**data["pattern"])
 
+    # ------------------------------------------------------------------
+    # Online evaluator endpoints
+    # ------------------------------------------------------------------
+
+    def create_online_evaluator(self, payload: dict) -> MonitorOnlineEvaluator:
+        data = self._request("POST", "/online-evaluators", json=self._with_workspace(payload))
+        return MonitorOnlineEvaluator(**data["evaluator"])
+
+    def list_online_evaluators(self) -> List[MonitorOnlineEvaluator]:
+        data = self._request("GET", "/online-evaluators", params=self._workspace_params())
+        return [MonitorOnlineEvaluator(**e) for e in data.get("evaluators", [])]
+
+    def get_online_evaluator(self, evaluator_id: str) -> MonitorOnlineEvaluator:
+        data = self._request(
+            "GET", f"/online-evaluators/{evaluator_id}", params=self._workspace_params()
+        )
+        return MonitorOnlineEvaluator(**data["evaluator"])
+
+    def update_online_evaluator(self, evaluator_id: str, payload: dict) -> MonitorOnlineEvaluator:
+        data = self._request(
+            "PUT", f"/online-evaluators/{evaluator_id}", json=self._with_workspace(payload)
+        )
+        return MonitorOnlineEvaluator(**data["evaluator"])
+
+    def delete_online_evaluator(self, evaluator_id: str) -> None:
+        self._request(
+            "DELETE", f"/online-evaluators/{evaluator_id}", params=self._workspace_params()
+        )
+
+    def get_online_evaluator_ratings(self, evaluator_id: str, window: str) -> List[OnlineEvaluatorRatingPoint]:
+        params = {**(self._workspace_params() or {}), "window": window}
+        data = self._request("GET", f"/online-evaluators/{evaluator_id}/ratings", params=params)
+        return [OnlineEvaluatorRatingPoint(**p) for p in data.get("points", [])]
+
+    def get_online_evaluator_events(self, evaluator_id: str, window: str) -> List[OnlineEvaluatorEvent]:
+        params = {**(self._workspace_params() or {}), "window": window}
+        data = self._request("GET", f"/online-evaluators/{evaluator_id}/events", params=params)
+        return [OnlineEvaluatorEvent(**e) for e in data.get("events", [])]
+
     # ------------------------------------------------------------------
     # Signal endpoints
     # ------------------------------------------------------------------

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx/monitor/models.py

@@ -37,6 +37,65 @@ class MonitorPattern(BaseModel):
         extra = "ignore"
 
 
+class MonitorOnlineEvaluator(BaseModel):
+    """Continuous LLM-judge scoring of a sample of live production traffic, distinct from a
+    MonitorPattern (which matches rules, not judgment). Built via
+    ``client.monitor.online_evaluators.builder(...).publish()``. References an
+    ``evaluation_settings_id`` (an Evaluator config: criteria, judge prompt, judge model) rather
+    than storing its own copy, the same config datasets/Evaluate runs use.
+
+    A score below ``alert_threshold`` raises/updates a Signal (``client.monitor.signals``), the
+    same triage surface a failing MonitorPattern already lands on, tagged with ``severity``. Set
+    ``alert_threshold=None`` to score without ever raising a Signal.
+    """
+
+    id: str = Field(alias="_id")
+    name: str
+    evaluation_settings_id: str = Field(alias="evaluationSettingsId")
+    sample_rate: float = Field(default=0.1, alias="sampleRate")
+    scope_mode: str = Field(default="all", alias="scopeMode")
+    agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
+    enabled: bool = True
+    alert_threshold: Optional[float] = Field(default=5, alias="alertThreshold")
+    severity: str = "medium"
+    created_at: Optional[str] = Field(default=None, alias="createdAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class OnlineEvaluatorRatingPoint(BaseModel):
+    """One bucket of a ratings-over-time series, see
+    ``client.monitor.online_evaluators.ratings(evaluator_id)``."""
+
+    label: str
+    ts: int
+    average_rating: Optional[float] = Field(default=None, alias="averageRating")
+    count: int = 0
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class OnlineEvaluatorEvent(BaseModel):
+    """One individually scored trace behind a point on the ratings series, worst-rated first,
+    see ``client.monitor.online_evaluators.events(evaluator_id)``."""
+
+    id: str
+    trace_id: str = Field(alias="traceId")
+    rating: float
+    justification: Optional[str] = None
+    created_at: str = Field(alias="createdAt")
+    input: str
+    output: str
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
 class SignalOccurrence(BaseModel):
     """One hit behind a signal, capped at the server's most recent N per signal."""
 

agentx_python-0.6.12/agentx/monitor/online_evaluators.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional, TYPE_CHECKING
+
+from agentx.monitor.models import MonitorOnlineEvaluator, OnlineEvaluatorRatingPoint, OnlineEvaluatorEvent
+
+if TYPE_CHECKING:
+    from agentx.monitor.client import MonitorClient
+
+logger = logging.getLogger(__name__)
+
+
+class MonitorOnlineEvaluatorBuilder:
+    """Fluent builder for creating an online evaluator: continuous LLM-judge scoring of a
+    sample of live production traffic, distinct from a MonitorPattern's rule matching.
+    ``evaluation_settings_id`` must reference an existing Evaluator config (criteria, judge
+    prompt, judge model), the same config datasets/Evaluate runs use, see
+    ``client.evaluations.settings.builder(...)``.
+    """
+
+    def __init__(
+        self,
+        client: "MonitorClient",
+        name: str,
+        evaluation_settings_id: str,
+        sample_rate: float = 0.1,
+        scope_mode: str = "all",
+        agent_ids: Optional[List[str]] = None,
+        enabled: bool = True,
+        alert_threshold: Optional[float] = 5,
+        severity: str = "medium",
+    ):
+        self._client = client
+        self._payload: Dict[str, Any] = {
+            "name": name,
+            "evaluationSettingsId": evaluation_settings_id,
+            # Every check here is a real LLM call against your own API key: keep this low
+            # unless you want to score every trace.
+            "sampleRate": sample_rate,
+            "scopeMode": scope_mode,
+            "agentIds": agent_ids or [],
+            "enabled": enabled,
+            # A score below this raises/updates a Signal, same triage surface a failing
+            # MonitorPattern already lands on. None scores without ever raising one.
+            "alertThreshold": alert_threshold,
+            "severity": severity,
+        }
+
+    def publish(self) -> MonitorOnlineEvaluator:
+        logger.info("Publishing online evaluator '%s'", self._payload["name"])
+        return self._client.create_online_evaluator(self._payload)
+
+
+class MonitorOnlineEvaluatorClient:
+    """Thin wrapper surfaced as ``client.monitor.online_evaluators``."""
+
+    def __init__(self, client: "MonitorClient"):
+        self._client = client
+
+    def builder(
+        self,
+        name: str,
+        evaluation_settings_id: str,
+        sample_rate: float = 0.1,
+        scope_mode: str = "all",
+        agent_ids: Optional[List[str]] = None,
+        enabled: bool = True,
+        alert_threshold: Optional[float] = 5,
+        severity: str = "medium",
+    ) -> MonitorOnlineEvaluatorBuilder:
+        return MonitorOnlineEvaluatorBuilder(
+            self._client,
+            name=name,
+            evaluation_settings_id=evaluation_settings_id,
+            sample_rate=sample_rate,
+            scope_mode=scope_mode,
+            agent_ids=agent_ids,
+            enabled=enabled,
+            alert_threshold=alert_threshold,
+            severity=severity,
+        )
+
+    def get(self, evaluator_id: str) -> MonitorOnlineEvaluator:
+        return self._client.get_online_evaluator(evaluator_id)
+
+    def list(self) -> List[MonitorOnlineEvaluator]:
+        return self._client.list_online_evaluators()
+
+    def update(self, evaluator_id: str, **fields: Any) -> MonitorOnlineEvaluator:
+        """Partial update: pass only the fields you want to change, e.g.
+        ``client.monitor.online_evaluators.update(id, enabled=False)`` to pause one, or
+        ``sample_rate=0.25`` to change its sampling. Field names match the builder's
+        (``evaluation_settings_id``, ``sample_rate``, ``scope_mode``, ``agent_ids``, ``enabled``,
+        ``alert_threshold``, ``severity``). Pass ``alert_threshold=None`` to stop this evaluator
+        from ever raising a Signal.
+        """
+        alias_map = {
+            "evaluation_settings_id": "evaluationSettingsId",
+            "sample_rate": "sampleRate",
+            "scope_mode": "scopeMode",
+            "agent_ids": "agentIds",
+            "alert_threshold": "alertThreshold",
+        }
+        payload = {alias_map.get(key, key): value for key, value in fields.items()}
+        return self._client.update_online_evaluator(evaluator_id, payload)
+
+    def delete(self, evaluator_id: str) -> None:
+        self._client.delete_online_evaluator(evaluator_id)
+
+    def ratings(self, evaluator_id: str, window: str = "7d") -> List[OnlineEvaluatorRatingPoint]:
+        """Bucketed average-rating-over-time for this evaluator. ``window`` is one of
+        ``"24h"``, ``"7d"``, ``"30d"``."""
+        return self._client.get_online_evaluator_ratings(evaluator_id, window)
+
+    def events(self, evaluator_id: str, window: str = "7d") -> List[OnlineEvaluatorEvent]:
+        """Individually scored traces behind the ratings series, worst-rated first and capped
+        (see the dashboard's Online Evaluators tab for the same view), lets a low point on the
+        ratings series be traced back to exactly which conversation(s) caused it and why."""
+        return self._client.get_online_evaluator_events(evaluator_id, window)

agentx_python-0.6.12/agentx/version.py

@@ -0,0 +1 @@
+VERSION = "0.6.12"

{agentx_python-0.6.11 → agentx_python-0.6.12/agentx_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentx-python
-Version: 0.6.11
+Version: 0.6.12
 Summary: Official Python SDK for AgentX (https://www.agentx.so/)
 Home-page: https://github.com/AgentX-ai/AgentX-python
 Author: Robin Wang and AgentX Team
@@ -298,6 +298,19 @@ Per-agent coverage/threshold settings (sample rate, retention, and threshold ove
 client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
 ```
 
+Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
+
+```python
+evaluator = client.monitor.online_evaluators.builder(
+    name="Helpfulness",
+    evaluation_settings_id=settings.id,
+    sample_rate=0.1,
+    alert_threshold=5,
+).publish()
+
+client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
+```
+
 See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
 
 ---
@@ -330,7 +343,7 @@ Ask a case's question several extra ways each run, LLM-paraphrased server-side,
 Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
 
 ```python
-prompt = client.evaluations.prompts.get("support-agent-system-prompt")
+prompt = client.evaluations.prompts.get("support-agent-system-prompt")  # or prompt.id
 # use prompt.text as your own agent's system prompt
 
 client.evaluations.run(

{agentx_python-0.6.11 → agentx_python-0.6.12}/agentx_python.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ agentx/integrations/openai_agents.py
 agentx/monitor/__init__.py
 agentx/monitor/client.py
 agentx/monitor/models.py
+agentx/monitor/online_evaluators.py
 agentx/monitor/patterns.py
 agentx/monitor/profile.py
 agentx/monitor/signals.py

agentx_python-0.6.11/agentx/version.py

@@ -1 +0,0 @@
-VERSION = "0.6.11"