agentx-python 0.6.5__tar.gz → 0.6.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.5 → agentx_python-0.6.6}/PKG-INFO +27 -1
- {agentx_python-0.6.5 → agentx_python-0.6.6}/README.md +26 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/agentx.py +10 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/crewai.py +7 -1
- agentx_python-0.6.6/agentx/monitor/__init__.py +10 -0
- agentx_python-0.6.6/agentx/monitor/client.py +134 -0
- agentx_python-0.6.6/agentx/monitor/models.py +37 -0
- agentx_python-0.6.6/agentx/monitor/patterns.py +121 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/tracing/tracer.py +30 -0
- agentx_python-0.6.6/agentx/version.py +1 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/PKG-INFO +27 -1
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/SOURCES.txt +4 -0
- agentx_python-0.6.5/agentx/version.py +0 -1
- {agentx_python-0.6.5 → agentx_python-0.6.6}/LICENSE +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/client.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/models.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/runner.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/exceptions.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/_perf.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/google_adk.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/langchain.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/integrations/openai_agents.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/py.typed +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx/util.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/setup.cfg +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/setup.py +0 -0
- {agentx_python-0.6.5 → agentx_python-0.6.6}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.6
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -70,6 +70,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
70
70
|
- [Chat (streaming and non-streaming)](#chat-streaming-and-non-streaming)
|
|
71
71
|
- [Workforce (multi-agent orchestration)](#workforce-multi-agent-orchestration) — teams of agents with a designated manager
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
|
+
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
73
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
74
75
|
- [Links](#links)
|
|
75
76
|
|
|
@@ -245,6 +246,30 @@ See **[TRACING.md](TRACING.md)** for the complete guide — session grouping, er
|
|
|
245
246
|
|
|
246
247
|
---
|
|
247
248
|
|
|
249
|
+
## Monitor
|
|
250
|
+
|
|
251
|
+
Automatic production monitoring: check traces against detection patterns and get back triage-ready signals. A **pattern** is a first-class SDK resource with a real id, just like a `Dataset` or `EvaluationSettings`: create one once, then reference it by id at trace time.
|
|
252
|
+
|
|
253
|
+
```python
|
|
254
|
+
pattern = client.monitor.patterns.builder(
|
|
255
|
+
name="Promises a refund",
|
|
256
|
+
detector_kind="semantic",
|
|
257
|
+
semantic_prompt="The response promises a refund.",
|
|
258
|
+
severity="high",
|
|
259
|
+
).publish()
|
|
260
|
+
|
|
261
|
+
with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]) as span:
|
|
262
|
+
span.output = call_llm(query)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`monitor=True` checks the trace immediately, no dashboard setup required. `pattern_ids` restricts detection to exactly those patterns; omit it to run the full default sweep instead (built-in checks like empty response, trace error, and latency regression, plus every pattern enabled for the workspace).
|
|
266
|
+
|
|
267
|
+
This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
|
|
268
|
+
|
|
269
|
+
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
248
273
|
## Custom agent evaluations
|
|
249
274
|
|
|
250
275
|
Evaluate **any** AI agent — LangChain, CrewAI, AutoGen, LlamaIndex, OpenAI, Anthropic, HTTP endpoints, or plain Python — using AgentX as the scoring and reporting backend. Includes optional **cosine** and **Jaccard** similarity metrics alongside LLM-graded ratings.
|
|
@@ -274,3 +299,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
274
299
|
- **PyPI** — [agentx-python](https://pypi.org/project/agentx-python/)
|
|
275
300
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
276
301
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
302
|
+
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
@@ -22,6 +22,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
22
22
|
- [Chat (streaming and non-streaming)](#chat-streaming-and-non-streaming)
|
|
23
23
|
- [Workforce (multi-agent orchestration)](#workforce-multi-agent-orchestration) — teams of agents with a designated manager
|
|
24
24
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
25
|
+
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
25
26
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
26
27
|
- [Links](#links)
|
|
27
28
|
|
|
@@ -197,6 +198,30 @@ See **[TRACING.md](TRACING.md)** for the complete guide — session grouping, er
|
|
|
197
198
|
|
|
198
199
|
---
|
|
199
200
|
|
|
201
|
+
## Monitor
|
|
202
|
+
|
|
203
|
+
Automatic production monitoring: check traces against detection patterns and get back triage-ready signals. A **pattern** is a first-class SDK resource with a real id, just like a `Dataset` or `EvaluationSettings`: create one once, then reference it by id at trace time.
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
pattern = client.monitor.patterns.builder(
|
|
207
|
+
name="Promises a refund",
|
|
208
|
+
detector_kind="semantic",
|
|
209
|
+
semantic_prompt="The response promises a refund.",
|
|
210
|
+
severity="high",
|
|
211
|
+
).publish()
|
|
212
|
+
|
|
213
|
+
with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]) as span:
|
|
214
|
+
span.output = call_llm(query)
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
`monitor=True` checks the trace immediately, no dashboard setup required. `pattern_ids` restricts detection to exactly those patterns; omit it to run the full default sweep instead (built-in checks like empty response, trace error, and latency regression, plus every pattern enabled for the workspace).
|
|
218
|
+
|
|
219
|
+
This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
|
|
220
|
+
|
|
221
|
+
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
200
225
|
## Custom agent evaluations
|
|
201
226
|
|
|
202
227
|
Evaluate **any** AI agent — LangChain, CrewAI, AutoGen, LlamaIndex, OpenAI, Anthropic, HTTP endpoints, or plain Python — using AgentX as the scoring and reporting backend. Includes optional **cosine** and **Jaccard** similarity metrics alongside LLM-graded ratings.
|
|
@@ -226,3 +251,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
226
251
|
- **PyPI** — [agentx-python](https://pypi.org/project/agentx-python/)
|
|
227
252
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
228
253
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
254
|
+
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
@@ -29,6 +29,7 @@ class AgentX:
|
|
|
29
29
|
|
|
30
30
|
from agentx.evaluations.client import EvaluationsClient
|
|
31
31
|
from agentx.evaluations.runner import EvaluationsRunner
|
|
32
|
+
from agentx.monitor.client import MonitorClient
|
|
32
33
|
from agentx.tracing.ingest_client import IngestClient
|
|
33
34
|
from agentx.tracing.tracer import Tracer
|
|
34
35
|
from agentx.version import VERSION
|
|
@@ -41,6 +42,15 @@ class AgentX:
|
|
|
41
42
|
)
|
|
42
43
|
self.evaluations = EvaluationsRunner(_eval_client)
|
|
43
44
|
|
|
45
|
+
# Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
|
|
46
|
+
# against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]).
|
|
47
|
+
self.monitor = MonitorClient(
|
|
48
|
+
api_key=self.api_key,
|
|
49
|
+
sdk_version=VERSION,
|
|
50
|
+
base_url=self.base_url,
|
|
51
|
+
workspace_id=self.workspace_id,
|
|
52
|
+
)
|
|
53
|
+
|
|
44
54
|
_ingest_client = IngestClient(
|
|
45
55
|
api_key=self.api_key,
|
|
46
56
|
sdk_version=VERSION,
|
|
@@ -121,8 +121,13 @@ class AgentXCrewObserver:
|
|
|
121
121
|
input: Optional[Any] = None,
|
|
122
122
|
metadata: Optional[Dict[str, Any]] = None,
|
|
123
123
|
session_id: Optional[str] = None,
|
|
124
|
+
sync: bool = False,
|
|
124
125
|
):
|
|
125
|
-
"""Return a context-manager span for manual kickoff wrapping.
|
|
126
|
+
"""Return a context-manager span for manual kickoff wrapping.
|
|
127
|
+
|
|
128
|
+
Pass ``sync=True`` to send synchronously so ``span.trace_id`` is populated once the
|
|
129
|
+
block exits — e.g. to attach the trace to an evaluation result. See ``Tracer.trace()``.
|
|
130
|
+
"""
|
|
126
131
|
from agentx.tracing.tracer import _TraceSpan
|
|
127
132
|
|
|
128
133
|
return _TraceSpan(
|
|
@@ -132,4 +137,5 @@ class AgentXCrewObserver:
|
|
|
132
137
|
metadata=metadata or self._metadata,
|
|
133
138
|
framework="crewai",
|
|
134
139
|
session_id=session_id or self._session_id,
|
|
140
|
+
sync=sync,
|
|
135
141
|
)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from agentx.monitor.client import MonitorClient
|
|
2
|
+
from agentx.monitor.models import MonitorPattern
|
|
3
|
+
from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"MonitorClient",
|
|
7
|
+
"MonitorPattern",
|
|
8
|
+
"MonitorPatternBuilder",
|
|
9
|
+
"MonitorPatternClient",
|
|
10
|
+
]
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any, List, Optional
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from agentx.monitor.models import MonitorPattern
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
from agentx.util import _DEFAULT_API_BASE as _UTIL_API_BASE
|
|
15
|
+
|
|
16
|
+
SDK_NAME = "agentx-python"
|
|
17
|
+
|
|
18
|
+
_RETRYABLE_STATUS = {429, 500, 502, 503, 504}
|
|
19
|
+
_MAX_RETRIES = 3
|
|
20
|
+
_RETRY_BACKOFF = [1.0, 2.0, 4.0]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AgentXMonitorError(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AgentXAuthError(AgentXMonitorError):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AgentXValidationError(AgentXMonitorError):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MonitorClient:
|
|
36
|
+
"""Low-level HTTP client for the Monitor API (``/monitor``). Accessed via
|
|
37
|
+
``client.monitor`` on the top-level :class:`agentx.AgentX` instance; most callers
|
|
38
|
+
should use ``client.monitor.patterns.builder(...)`` instead of this directly."""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
api_key: str,
|
|
43
|
+
sdk_version: str = "unknown",
|
|
44
|
+
base_url: Optional[str] = None,
|
|
45
|
+
workspace_id: Optional[str] = None,
|
|
46
|
+
):
|
|
47
|
+
if not api_key:
|
|
48
|
+
raise AgentXAuthError("AGENTX_API_KEY is required")
|
|
49
|
+
self._api_key = api_key
|
|
50
|
+
self._sdk_version = sdk_version
|
|
51
|
+
# Falls back to the caller's default workspace server-side when unset — mirrors
|
|
52
|
+
# EvaluationsClient. Without this, pattern creation silently lands in whatever
|
|
53
|
+
# workspace the API key's user defaults to, not the one the caller intended.
|
|
54
|
+
self._workspace_id = workspace_id
|
|
55
|
+
_api_base = (
|
|
56
|
+
base_url or os.getenv("AGENTX_API_BASE_URL", _UTIL_API_BASE)
|
|
57
|
+
).rstrip("/")
|
|
58
|
+
if not _api_base.endswith("/monitor"):
|
|
59
|
+
_api_base = f"{_api_base}/monitor"
|
|
60
|
+
self._base_url = _api_base
|
|
61
|
+
self._session = requests.Session()
|
|
62
|
+
self._session.headers.update(
|
|
63
|
+
{
|
|
64
|
+
"x-api-key": self._api_key,
|
|
65
|
+
"Content-Type": "application/json",
|
|
66
|
+
"User-Agent": f"{SDK_NAME}/{self._sdk_version}",
|
|
67
|
+
"accept": "*/*",
|
|
68
|
+
}
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
from agentx.monitor.patterns import MonitorPatternClient
|
|
72
|
+
|
|
73
|
+
self.patterns = MonitorPatternClient(self)
|
|
74
|
+
|
|
75
|
+
# ------------------------------------------------------------------
|
|
76
|
+
# Low-level HTTP
|
|
77
|
+
# ------------------------------------------------------------------
|
|
78
|
+
|
|
79
|
+
def _with_workspace(self, payload: dict) -> dict:
|
|
80
|
+
if self._workspace_id and not payload.get("workspaceId"):
|
|
81
|
+
return {**payload, "workspaceId": self._workspace_id}
|
|
82
|
+
return payload
|
|
83
|
+
|
|
84
|
+
def _workspace_params(self) -> Optional[dict]:
|
|
85
|
+
return {"workspaceId": self._workspace_id} if self._workspace_id else None
|
|
86
|
+
|
|
87
|
+
def _request(self, method: str, path: str, timeout: int = 30, **kwargs) -> Any:
|
|
88
|
+
url = f"{self._base_url}{path}"
|
|
89
|
+
last_exc: Optional[Exception] = None
|
|
90
|
+
for attempt, wait in enumerate([0.0] + _RETRY_BACKOFF):
|
|
91
|
+
if wait:
|
|
92
|
+
time.sleep(wait)
|
|
93
|
+
try:
|
|
94
|
+
resp = self._session.request(method, url, timeout=timeout, **kwargs)
|
|
95
|
+
except requests.RequestException as e:
|
|
96
|
+
last_exc = e
|
|
97
|
+
logger.debug("Request error (attempt %d): %s", attempt + 1, e)
|
|
98
|
+
continue
|
|
99
|
+
|
|
100
|
+
if resp.status_code == 401:
|
|
101
|
+
raise AgentXAuthError("Invalid or missing API key")
|
|
102
|
+
if resp.status_code == 422:
|
|
103
|
+
raise AgentXValidationError(resp.text)
|
|
104
|
+
if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES - 1:
|
|
105
|
+
logger.debug(
|
|
106
|
+
"Retryable status %d (attempt %d)", resp.status_code, attempt + 1
|
|
107
|
+
)
|
|
108
|
+
last_exc = AgentXMonitorError(f"HTTP {resp.status_code}")
|
|
109
|
+
continue
|
|
110
|
+
if not resp.ok:
|
|
111
|
+
raise AgentXMonitorError(f"HTTP {resp.status_code}: {resp.text}")
|
|
112
|
+
try:
|
|
113
|
+
return resp.json()
|
|
114
|
+
except Exception:
|
|
115
|
+
return resp.text
|
|
116
|
+
raise AgentXMonitorError(f"Request failed after retries: {last_exc}")
|
|
117
|
+
|
|
118
|
+
# ------------------------------------------------------------------
|
|
119
|
+
# Pattern endpoints
|
|
120
|
+
# ------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
def create_pattern(self, payload: dict) -> MonitorPattern:
|
|
123
|
+
data = self._request("POST", "/patterns", json=self._with_workspace(payload))
|
|
124
|
+
return MonitorPattern(**data["pattern"])
|
|
125
|
+
|
|
126
|
+
def list_patterns(self) -> List[MonitorPattern]:
|
|
127
|
+
data = self._request("GET", "/patterns", params=self._workspace_params())
|
|
128
|
+
return [MonitorPattern(**p) for p in data.get("patterns", [])]
|
|
129
|
+
|
|
130
|
+
def get_pattern(self, pattern_id: str) -> MonitorPattern:
|
|
131
|
+
data = self._request(
|
|
132
|
+
"GET", f"/patterns/{pattern_id}", params=self._workspace_params()
|
|
133
|
+
)
|
|
134
|
+
return MonitorPattern(**data["pattern"])
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class MonitorPattern(BaseModel):
|
|
8
|
+
"""A detection rule checked against production traces. Built via
|
|
9
|
+
``client.monitor.patterns.builder(...).publish()`` and referenced by id as a
|
|
10
|
+
``pattern_ids`` entry in ``tracer.trace(..., monitor=True, pattern_ids=[...])``.
|
|
11
|
+
|
|
12
|
+
A "failure" pattern (the default) raises a signal to triage; a "proper" pattern logs a
|
|
13
|
+
healthy tally instead. Only one of ``include_terms``/``regex``/``semantic_prompt`` is
|
|
14
|
+
meaningful at a time, selected by ``detector_kind``.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
id: str = Field(alias="_id")
|
|
18
|
+
key: str
|
|
19
|
+
name: str
|
|
20
|
+
description: Optional[str] = None
|
|
21
|
+
category: Optional[str] = None
|
|
22
|
+
detector_kind: str = Field(default="contains", alias="detectorKind")
|
|
23
|
+
match_target: List[str] = Field(default_factory=lambda: ["response"], alias="matchTarget")
|
|
24
|
+
match_mode: str = Field(default="any", alias="matchMode")
|
|
25
|
+
include_terms: List[str] = Field(default_factory=list, alias="includeTerms")
|
|
26
|
+
exclude_terms: List[str] = Field(default_factory=list, alias="excludeTerms")
|
|
27
|
+
regex: Optional[str] = None
|
|
28
|
+
semantic_prompt: Optional[str] = Field(default=None, alias="semanticPrompt")
|
|
29
|
+
severity: str = "medium"
|
|
30
|
+
polarity: str = "failure"
|
|
31
|
+
enabled: bool = True
|
|
32
|
+
sample_rate: float = Field(default=1.0, alias="sampleRate")
|
|
33
|
+
scope_mode: str = Field(default="all", alias="scopeMode")
|
|
34
|
+
agent_ids: List[str] = Field(default_factory=list, alias="agentIds")
|
|
35
|
+
|
|
36
|
+
class Config:
|
|
37
|
+
extra = "ignore"
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from agentx.monitor.models import MonitorPattern
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from agentx.monitor.client import MonitorClient
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MonitorPatternBuilder:
|
|
15
|
+
"""Fluent builder for creating a custom pattern. ``detector_kind`` selects which of
|
|
16
|
+
``include_terms``/``regex``/``semantic_prompt`` is used:
|
|
17
|
+
|
|
18
|
+
- ``"contains"`` (default): ``include_terms`` — a match if any (or all, with
|
|
19
|
+
``match_mode="all"``) phrase appears in the target text.
|
|
20
|
+
- ``"regex"``: ``regex`` — a single regular expression.
|
|
21
|
+
- ``"semantic"``: ``semantic_prompt`` — an LLM judges whether the response violates the
|
|
22
|
+
described rubric.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
client: "MonitorClient",
|
|
28
|
+
name: str,
|
|
29
|
+
description: Optional[str] = None,
|
|
30
|
+
category: Optional[str] = None,
|
|
31
|
+
detector_kind: str = "contains",
|
|
32
|
+
match_target: Optional[List[str]] = None,
|
|
33
|
+
match_mode: str = "any",
|
|
34
|
+
include_terms: Optional[List[str]] = None,
|
|
35
|
+
exclude_terms: Optional[List[str]] = None,
|
|
36
|
+
regex: Optional[str] = None,
|
|
37
|
+
semantic_prompt: Optional[str] = None,
|
|
38
|
+
severity: str = "medium",
|
|
39
|
+
polarity: str = "failure",
|
|
40
|
+
enabled: bool = True,
|
|
41
|
+
sample_rate: float = 1.0,
|
|
42
|
+
scope_mode: str = "all",
|
|
43
|
+
agent_ids: Optional[List[str]] = None,
|
|
44
|
+
):
|
|
45
|
+
self._client = client
|
|
46
|
+
self._payload: Dict[str, Any] = {
|
|
47
|
+
"name": name,
|
|
48
|
+
"description": description,
|
|
49
|
+
"category": category,
|
|
50
|
+
"detectorKind": detector_kind,
|
|
51
|
+
"matchTarget": match_target or ["response"],
|
|
52
|
+
"matchMode": match_mode,
|
|
53
|
+
"includeTerms": include_terms or [],
|
|
54
|
+
"excludeTerms": exclude_terms or [],
|
|
55
|
+
"regex": regex,
|
|
56
|
+
"semanticPrompt": semantic_prompt,
|
|
57
|
+
"severity": severity,
|
|
58
|
+
# A "failure" pattern (default) raises a signal to triage; a "proper" pattern logs
|
|
59
|
+
# a healthy tally instead.
|
|
60
|
+
"polarity": polarity,
|
|
61
|
+
"enabled": enabled,
|
|
62
|
+
"sampleRate": sample_rate,
|
|
63
|
+
"scopeMode": scope_mode,
|
|
64
|
+
"agentIds": agent_ids or [],
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
def publish(self) -> MonitorPattern:
|
|
68
|
+
logger.info("Publishing monitor pattern '%s'", self._payload["name"])
|
|
69
|
+
return self._client.create_pattern(self._payload)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class MonitorPatternClient:
|
|
73
|
+
"""Thin wrapper surfaced as ``client.monitor.patterns``."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, client: "MonitorClient"):
|
|
76
|
+
self._client = client
|
|
77
|
+
|
|
78
|
+
def builder(
|
|
79
|
+
self,
|
|
80
|
+
name: str,
|
|
81
|
+
description: Optional[str] = None,
|
|
82
|
+
category: Optional[str] = None,
|
|
83
|
+
detector_kind: str = "contains",
|
|
84
|
+
match_target: Optional[List[str]] = None,
|
|
85
|
+
match_mode: str = "any",
|
|
86
|
+
include_terms: Optional[List[str]] = None,
|
|
87
|
+
exclude_terms: Optional[List[str]] = None,
|
|
88
|
+
regex: Optional[str] = None,
|
|
89
|
+
semantic_prompt: Optional[str] = None,
|
|
90
|
+
severity: str = "medium",
|
|
91
|
+
polarity: str = "failure",
|
|
92
|
+
enabled: bool = True,
|
|
93
|
+
sample_rate: float = 1.0,
|
|
94
|
+
scope_mode: str = "all",
|
|
95
|
+
agent_ids: Optional[List[str]] = None,
|
|
96
|
+
) -> MonitorPatternBuilder:
|
|
97
|
+
return MonitorPatternBuilder(
|
|
98
|
+
self._client,
|
|
99
|
+
name=name,
|
|
100
|
+
description=description,
|
|
101
|
+
category=category,
|
|
102
|
+
detector_kind=detector_kind,
|
|
103
|
+
match_target=match_target,
|
|
104
|
+
match_mode=match_mode,
|
|
105
|
+
include_terms=include_terms,
|
|
106
|
+
exclude_terms=exclude_terms,
|
|
107
|
+
regex=regex,
|
|
108
|
+
semantic_prompt=semantic_prompt,
|
|
109
|
+
severity=severity,
|
|
110
|
+
polarity=polarity,
|
|
111
|
+
enabled=enabled,
|
|
112
|
+
sample_rate=sample_rate,
|
|
113
|
+
scope_mode=scope_mode,
|
|
114
|
+
agent_ids=agent_ids,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def get(self, pattern_id: str) -> MonitorPattern:
|
|
118
|
+
return self._client.get_pattern(pattern_id)
|
|
119
|
+
|
|
120
|
+
def list(self) -> List[MonitorPattern]:
|
|
121
|
+
return self._client.list_patterns()
|
|
@@ -78,6 +78,8 @@ class _TraceSpan:
|
|
|
78
78
|
model: Optional[str] = None,
|
|
79
79
|
session_id: Optional[str] = None,
|
|
80
80
|
sync: bool = False,
|
|
81
|
+
monitor: bool = False,
|
|
82
|
+
pattern_ids: Optional[List[str]] = None,
|
|
81
83
|
) -> None:
|
|
82
84
|
self._tracer = tracer
|
|
83
85
|
self.name = name
|
|
@@ -91,6 +93,11 @@ class _TraceSpan:
|
|
|
91
93
|
# is populated by the time the `with` block exits — see Tracer.trace()'s sync param.
|
|
92
94
|
self._sync = sync
|
|
93
95
|
self._trace_id: Optional[str] = None
|
|
96
|
+
# Monitor: check this trace against patterns immediately on ingest, no dashboard profile
|
|
97
|
+
# required. pattern_ids (if given) fully defines what's checked — only those patterns run,
|
|
98
|
+
# the built-in checks are skipped. See Tracer.trace()'s monitor/pattern_ids params.
|
|
99
|
+
self._monitor = monitor
|
|
100
|
+
self._pattern_ids = pattern_ids
|
|
94
101
|
|
|
95
102
|
# Fields the caller can set while inside the context manager
|
|
96
103
|
self.output: Any = None
|
|
@@ -171,6 +178,8 @@ class _TraceSpan:
|
|
|
171
178
|
|
|
172
179
|
self._trace_id = self._tracer._send(
|
|
173
180
|
sync=self._sync,
|
|
181
|
+
monitor=self._monitor or None,
|
|
182
|
+
pattern_ids=self._pattern_ids,
|
|
174
183
|
name=self.name,
|
|
175
184
|
input=_safe_serialize(self.input) if self.input is not None else None,
|
|
176
185
|
output=_safe_serialize(self.output) if self.output is not None else None,
|
|
@@ -615,6 +624,8 @@ class Tracer:
|
|
|
615
624
|
model: Optional[str] = None,
|
|
616
625
|
session_id: Optional[str] = None,
|
|
617
626
|
sync: bool = False,
|
|
627
|
+
monitor: bool = False,
|
|
628
|
+
pattern_ids: Optional[List[str]] = None,
|
|
618
629
|
) -> _TraceSpan:
|
|
619
630
|
"""
|
|
620
631
|
Return a :class:`_TraceSpan` that works as both a decorator and a
|
|
@@ -629,6 +640,19 @@ class Tracer:
|
|
|
629
640
|
resp = call_llm(...)
|
|
630
641
|
span.output = resp
|
|
631
642
|
return {"output": resp, "trace_id": span.trace_id}
|
|
643
|
+
|
|
644
|
+
Pass ``monitor=True`` to check this trace against Monitor patterns immediately, with no
|
|
645
|
+
dashboard profile required. ``pattern_ids`` (ids from ``client.monitor.patterns.builder(
|
|
646
|
+
...).publish()``) restricts detection to exactly those patterns; omit it to run the full
|
|
647
|
+
default sweep (built-in checks plus every enabled workspace pattern)::
|
|
648
|
+
|
|
649
|
+
pattern = client.monitor.patterns.builder(
|
|
650
|
+
name="Promises a refund", detector_kind="semantic",
|
|
651
|
+
semantic_prompt="The response promises a refund.",
|
|
652
|
+
).publish()
|
|
653
|
+
|
|
654
|
+
with client.tracer.trace("support_agent_call", monitor=True, pattern_ids=[pattern.id]) as span:
|
|
655
|
+
span.output = call_llm(...)
|
|
632
656
|
"""
|
|
633
657
|
return _TraceSpan(
|
|
634
658
|
tracer=self,
|
|
@@ -639,6 +663,8 @@ class Tracer:
|
|
|
639
663
|
model=model,
|
|
640
664
|
session_id=session_id,
|
|
641
665
|
sync=sync,
|
|
666
|
+
monitor=monitor,
|
|
667
|
+
pattern_ids=pattern_ids,
|
|
642
668
|
)
|
|
643
669
|
|
|
644
670
|
def flush(self, timeout: float = 5.0) -> None:
|
|
@@ -862,6 +888,10 @@ class Tracer:
|
|
|
862
888
|
wire["input_tokens"] = payload["input_tokens"]
|
|
863
889
|
if "output_tokens" in payload:
|
|
864
890
|
wire["output_tokens"] = payload["output_tokens"]
|
|
891
|
+
if "monitor" in payload:
|
|
892
|
+
wire["monitor"] = payload["monitor"]
|
|
893
|
+
if "pattern_ids" in payload:
|
|
894
|
+
wire["pattern_ids"] = payload["pattern_ids"]
|
|
865
895
|
|
|
866
896
|
pending_retrievals, self._pending_retrievals = self._pending_retrievals, []
|
|
867
897
|
pending_tool_calls, self._pending_tool_calls = self._pending_tool_calls, []
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = "0.6.6"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.6
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -70,6 +70,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
70
70
|
- [Chat (streaming and non-streaming)](#chat-streaming-and-non-streaming)
|
|
71
71
|
- [Workforce (multi-agent orchestration)](#workforce-multi-agent-orchestration) — teams of agents with a designated manager
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
|
+
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
73
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
74
75
|
- [Links](#links)
|
|
75
76
|
|
|
@@ -245,6 +246,30 @@ See **[TRACING.md](TRACING.md)** for the complete guide — session grouping, er
|
|
|
245
246
|
|
|
246
247
|
---
|
|
247
248
|
|
|
249
|
+
## Monitor
|
|
250
|
+
|
|
251
|
+
Automatic production monitoring: check traces against detection patterns and get back triage-ready signals. A **pattern** is a first-class SDK resource with a real id, just like a `Dataset` or `EvaluationSettings`: create one once, then reference it by id at trace time.
|
|
252
|
+
|
|
253
|
+
```python
|
|
254
|
+
pattern = client.monitor.patterns.builder(
|
|
255
|
+
name="Promises a refund",
|
|
256
|
+
detector_kind="semantic",
|
|
257
|
+
semantic_prompt="The response promises a refund.",
|
|
258
|
+
severity="high",
|
|
259
|
+
).publish()
|
|
260
|
+
|
|
261
|
+
with client.tracer.trace("support-agent", monitor=True, pattern_ids=[pattern.id]) as span:
|
|
262
|
+
span.output = call_llm(query)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`monitor=True` checks the trace immediately, no dashboard setup required. `pattern_ids` restricts detection to exactly those patterns; omit it to run the full default sweep instead (built-in checks like empty response, trace error, and latency regression, plus every pattern enabled for the workspace).
|
|
266
|
+
|
|
267
|
+
This works independently of the dashboard's per-agent monitoring toggle (Governance > Observe > Agents), which still auto-checks every trace from an agent once enabled there, with no code changes needed either way.
|
|
268
|
+
|
|
269
|
+
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
248
273
|
## Custom agent evaluations
|
|
249
274
|
|
|
250
275
|
Evaluate **any** AI agent — LangChain, CrewAI, AutoGen, LlamaIndex, OpenAI, Anthropic, HTTP endpoints, or plain Python — using AgentX as the scoring and reporting backend. Includes optional **cosine** and **Jaccard** similarity metrics alongside LLM-graded ratings.
|
|
@@ -274,3 +299,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
274
299
|
- **PyPI** — [agentx-python](https://pypi.org/project/agentx-python/)
|
|
275
300
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
276
301
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
302
|
+
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
@@ -30,6 +30,10 @@ agentx/integrations/google_adk.py
|
|
|
30
30
|
agentx/integrations/google_genai.py
|
|
31
31
|
agentx/integrations/langchain.py
|
|
32
32
|
agentx/integrations/openai_agents.py
|
|
33
|
+
agentx/monitor/__init__.py
|
|
34
|
+
agentx/monitor/client.py
|
|
35
|
+
agentx/monitor/models.py
|
|
36
|
+
agentx/monitor/patterns.py
|
|
33
37
|
agentx/resources/__init__.py
|
|
34
38
|
agentx/resources/agent.py
|
|
35
39
|
agentx/resources/conversation.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
VERSION = "0.6.5"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|