agentx-python 0.6.8__tar.gz → 0.6.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.8/agentx_python.egg-info → agentx_python-0.6.10}/PKG-INFO +28 -1
- {agentx_python-0.6.8 → agentx_python-0.6.10}/README.md +27 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/agentx.py +3 -1
- agentx_python-0.6.10/agentx/cli.py +158 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/langchain.py +84 -62
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/monitor/__init__.py +4 -1
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/monitor/client.py +20 -1
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/monitor/models.py +30 -0
- agentx_python-0.6.10/agentx/monitor/profile.py +68 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/tracing/tracer.py +7 -1
- agentx_python-0.6.10/agentx/version.py +1 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10/agentx_python.egg-info}/PKG-INFO +28 -1
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx_python.egg-info/SOURCES.txt +3 -0
- agentx_python-0.6.10/agentx_python.egg-info/entry_points.txt +2 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/setup.py +6 -0
- agentx_python-0.6.8/agentx/version.py +0 -1
- {agentx_python-0.6.8 → agentx_python-0.6.10}/LICENSE +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/client.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/models.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/runner.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/exceptions.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/_perf.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/crewai.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/google_adk.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/integrations/openai_agents.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/py.typed +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx/util.py +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/setup.cfg +0 -0
- {agentx_python-0.6.8 → agentx_python-0.6.10}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.10
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -72,6 +72,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
73
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
74
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
75
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
75
76
|
- [Links](#links)
|
|
76
77
|
|
|
77
78
|
---
|
|
@@ -273,6 +274,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
273
274
|
print(signal.summary, signal.occurrence_count)
|
|
274
275
|
```
|
|
275
276
|
|
|
277
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
281
|
+
```
|
|
282
|
+
|
|
276
283
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
277
284
|
|
|
278
285
|
---
|
|
@@ -306,6 +313,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
306
313
|
|
|
307
314
|
---
|
|
308
315
|
|
|
316
|
+
## Self-host
|
|
317
|
+
|
|
318
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
agentx-trace-eval --dev
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
328
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
309
335
|
## Links
|
|
310
336
|
|
|
311
337
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -314,3 +340,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
314
340
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
315
341
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
316
342
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
343
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -24,6 +24,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
24
24
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
25
25
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
26
26
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
27
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
27
28
|
- [Links](#links)
|
|
28
29
|
|
|
29
30
|
---
|
|
@@ -225,6 +226,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
225
226
|
print(signal.summary, signal.occurrence_count)
|
|
226
227
|
```
|
|
227
228
|
|
|
229
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
233
|
+
```
|
|
234
|
+
|
|
228
235
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
229
236
|
|
|
230
237
|
---
|
|
@@ -258,6 +265,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
258
265
|
|
|
259
266
|
---
|
|
260
267
|
|
|
268
|
+
## Self-host
|
|
269
|
+
|
|
270
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
agentx-trace-eval --dev
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
280
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
284
|
+
|
|
285
|
+
---
|
|
286
|
+
|
|
261
287
|
## Links
|
|
262
288
|
|
|
263
289
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -266,3 +292,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
266
292
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
267
293
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
268
294
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
295
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -44,7 +44,9 @@ class AgentX:
|
|
|
44
44
|
|
|
45
45
|
# Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
|
|
46
46
|
# against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]), then read
|
|
47
|
-
# back the resulting alerts/findings with client.monitor.signals.
|
|
47
|
+
# back the resulting alerts/findings with client.monitor.signals. Per-agent coverage and
|
|
48
|
+
# detection settings (sample rate, retention, threshold overrides like the built-in
|
|
49
|
+
# "Latency regression" pattern's threshold) are client.monitor.profile.
|
|
48
50
|
self.monitor = MonitorClient(
|
|
49
51
|
api_key=self.api_key,
|
|
50
52
|
sdk_version=VERSION,
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""
|
|
2
|
+
`agentx-trace-eval` — thin launcher for AgentX's self-hostable governance engine (Trace,
|
|
3
|
+
Evaluate, Monitor), published separately at github.com/AgentX-ai/AgentX-trace-eval (a Go CLI
|
|
4
|
+
wrapping a Bun-compiled TypeScript engine, not Python). That compiled engine binary is tens of
|
|
5
|
+
megabytes; most `pip install agentx-python` installs are just this SDK talking to the hosted
|
|
6
|
+
AgentX SaaS and would never touch it, so it isn't bundled in this package. Instead, this command
|
|
7
|
+
downloads the matching release into ~/.agentx/bin the first time it's needed (mirroring
|
|
8
|
+
AgentX-trace-eval's own install.sh) and then hands off to the real `agentx-server` binary.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
agentx-trace-eval --dev
|
|
12
|
+
agentx-trace-eval --port 5000 --db-url postgres://...
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import platform
|
|
17
|
+
import shutil
|
|
18
|
+
import stat
|
|
19
|
+
import sys
|
|
20
|
+
import tarfile
|
|
21
|
+
import tempfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Tuple
|
|
24
|
+
|
|
25
|
+
import requests
|
|
26
|
+
|
|
27
|
+
REPO = "AgentX-ai/AgentX-trace-eval"
|
|
28
|
+
INSTALL_DIR = Path(os.environ.get("AGENTX_INSTALL_DIR", str(Path.home() / ".agentx" / "bin")))
|
|
29
|
+
_BIN_NAMES = ("agentx", "agentx-server", "agentx-engine")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _platform_tag() -> Tuple[str, str]:
|
|
33
|
+
system = platform.system()
|
|
34
|
+
if system == "Darwin":
|
|
35
|
+
os_name = "darwin"
|
|
36
|
+
elif system == "Linux":
|
|
37
|
+
os_name = "linux"
|
|
38
|
+
else:
|
|
39
|
+
raise SystemExit(
|
|
40
|
+
f"agentx-trace-eval: unsupported OS {system!r} (self-host currently supports macOS and Linux)"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
machine = platform.machine().lower()
|
|
44
|
+
if machine in ("arm64", "aarch64"):
|
|
45
|
+
arch = "arm64"
|
|
46
|
+
elif machine in ("x86_64", "amd64"):
|
|
47
|
+
arch = "amd64"
|
|
48
|
+
else:
|
|
49
|
+
raise SystemExit(f"agentx-trace-eval: unsupported architecture {machine!r}")
|
|
50
|
+
|
|
51
|
+
return os_name, arch
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _release_url(asset: str, version: str) -> str:
|
|
55
|
+
if version == "latest":
|
|
56
|
+
return f"https://github.com/{REPO}/releases/latest/download/{asset}"
|
|
57
|
+
return f"https://github.com/{REPO}/releases/download/{version}/{asset}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _download(url: str, dest: Path) -> None:
|
|
61
|
+
response = requests.get(url, stream=True, timeout=60)
|
|
62
|
+
response.raise_for_status()
|
|
63
|
+
with open(dest, "wb") as f:
|
|
64
|
+
for chunk in response.iter_content(chunk_size=1 << 16):
|
|
65
|
+
f.write(chunk)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _extract_tar(archive: Path, dest_dir: Path) -> None:
|
|
69
|
+
with tarfile.open(archive) as tar:
|
|
70
|
+
try:
|
|
71
|
+
# filter="data" (PEP 706, Python 3.12+) rejects absolute paths/symlink escapes.
|
|
72
|
+
# Belt-and-suspenders here since these are trusted release assets built by our own CI
|
|
73
|
+
# (see REPO above), not arbitrary user-supplied archives.
|
|
74
|
+
tar.extractall(dest_dir, filter="data")
|
|
75
|
+
except TypeError:
|
|
76
|
+
tar.extractall(dest_dir) # Python < 3.12: filter kwarg doesn't exist yet
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _install(version: str = "latest") -> None:
|
|
80
|
+
os_name, arch = _platform_tag()
|
|
81
|
+
INSTALL_DIR.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
|
|
83
|
+
print(f"agentx-trace-eval: downloading agentx ({os_name}/{arch})...", file=sys.stderr)
|
|
84
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
85
|
+
tmp_dir = Path(tmp)
|
|
86
|
+
archive = tmp_dir / "agentx.tar.gz"
|
|
87
|
+
url = _release_url(f"agentx_{os_name}_{arch}.tar.gz", version)
|
|
88
|
+
try:
|
|
89
|
+
_download(url, archive)
|
|
90
|
+
except requests.HTTPError as exc:
|
|
91
|
+
raise SystemExit(
|
|
92
|
+
f"agentx-trace-eval: failed to download {url} ({exc}).\n"
|
|
93
|
+
"No published release found; see AgentX-trace-eval's README for building from source."
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
_extract_tar(archive, tmp_dir)
|
|
97
|
+
|
|
98
|
+
found_any = False
|
|
99
|
+
for name in _BIN_NAMES:
|
|
100
|
+
src = tmp_dir / name
|
|
101
|
+
if not src.exists():
|
|
102
|
+
continue
|
|
103
|
+
found_any = True
|
|
104
|
+
dest = INSTALL_DIR / name
|
|
105
|
+
shutil.move(str(src), str(dest))
|
|
106
|
+
dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
|
|
107
|
+
|
|
108
|
+
if not found_any:
|
|
109
|
+
raise SystemExit(f"agentx-trace-eval: downloaded archive from {url} didn't contain any of {_BIN_NAMES}")
|
|
110
|
+
|
|
111
|
+
if os.environ.get("AGENTX_TRACE_EVAL_SKIP_WEB"):
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
# Best-effort: the dashboard is a separate, platform-independent asset (see
|
|
115
|
+
# AgentX-trace-eval's README's "Dashboard release"). Missing it shouldn't block getting the
|
|
116
|
+
# engine running headless, so a failure here warns and continues rather than raising.
|
|
117
|
+
print("agentx-trace-eval: downloading dashboard...", file=sys.stderr)
|
|
118
|
+
web_dir = INSTALL_DIR / "web"
|
|
119
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
120
|
+
web_archive = Path(tmp) / "agentx-web.tar.gz"
|
|
121
|
+
web_url = _release_url("agentx-web.tar.gz", version)
|
|
122
|
+
try:
|
|
123
|
+
_download(web_url, web_archive)
|
|
124
|
+
except requests.HTTPError:
|
|
125
|
+
print(
|
|
126
|
+
f"agentx-trace-eval: no dashboard bundle found at {web_url}, continuing without one",
|
|
127
|
+
file=sys.stderr,
|
|
128
|
+
)
|
|
129
|
+
return
|
|
130
|
+
if web_dir.exists():
|
|
131
|
+
shutil.rmtree(web_dir)
|
|
132
|
+
web_dir.mkdir(parents=True)
|
|
133
|
+
_extract_tar(web_archive, web_dir)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def ensure_installed(version: str = "latest") -> Path:
|
|
137
|
+
"""Downloads agentx-server (+ its engine) into ~/.agentx/bin if not already present there.
|
|
138
|
+
Returns the path to the agentx-server executable. Set AGENTX_INSTALL_DIR to change where
|
|
139
|
+
this looks/installs; set AGENTX_TRACE_EVAL_VERSION to pin a release tag instead of latest."""
|
|
140
|
+
server_path = INSTALL_DIR / "agentx-server"
|
|
141
|
+
if not server_path.exists():
|
|
142
|
+
_install(version=version)
|
|
143
|
+
return server_path
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def main() -> None:
|
|
147
|
+
version = os.environ.get("AGENTX_TRACE_EVAL_VERSION", "latest")
|
|
148
|
+
server_path = ensure_installed(version=version)
|
|
149
|
+
if not server_path.exists():
|
|
150
|
+
raise SystemExit(f"agentx-trace-eval: {server_path} still missing after install, giving up")
|
|
151
|
+
|
|
152
|
+
# os.execv replaces this process rather than spawning a subprocess: signals, stdio, and the
|
|
153
|
+
# exit code all pass straight through to agentx-server, same as invoking it directly.
|
|
154
|
+
os.execv(str(server_path), [str(server_path)] + sys.argv[1:])
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
main()
|
|
@@ -20,6 +20,7 @@ Requires: ``pip install "agentx-python[langchain]"``
|
|
|
20
20
|
"""
|
|
21
21
|
from __future__ import annotations
|
|
22
22
|
|
|
23
|
+
import threading
|
|
23
24
|
import time
|
|
24
25
|
from typing import Any, Dict, List, Optional, Union
|
|
25
26
|
from uuid import UUID
|
|
@@ -262,6 +263,13 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
262
263
|
self._pending_retrieval_steps: List[Dict[str, Any]] = []
|
|
263
264
|
# run_id → {"start": float, "query": str}
|
|
264
265
|
self._retrieval_starts: Dict[UUID, Dict[str, Any]] = {}
|
|
266
|
+
# Guards appends to a top-level run's shared aggregate lists (tool_calls,
|
|
267
|
+
# perf_tool_calls, execution_steps, retrieval_steps). LangGraph's ToolNode
|
|
268
|
+
# runs multiple tool calls from one AIMessage concurrently via a thread
|
|
269
|
+
# pool (see langgraph.prebuilt.tool_node.ToolNode._func), so on_tool_end /
|
|
270
|
+
# on_tool_error can fire from several threads at once for the same
|
|
271
|
+
# top-level run.
|
|
272
|
+
self._state_lock = threading.Lock()
|
|
265
273
|
|
|
266
274
|
# ------------------------------------------------------------------
|
|
267
275
|
# Chain lifecycle
|
|
@@ -311,14 +319,14 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
311
319
|
return
|
|
312
320
|
latency_ms = int((time.time() - state["start"]) * 1000)
|
|
313
321
|
output = _extract_output(outputs)
|
|
322
|
+
# Each tool_call dict already carries its own start_time/end_time (set in
|
|
323
|
+
# on_tool_end/on_tool_error), so no re-pairing against perf_tool_calls by
|
|
324
|
+
# index is needed here. That used to be done via zip(), which silently
|
|
325
|
+
# mispaired timestamps when LangGraph's ToolNode ran several tool calls
|
|
326
|
+
# from one AIMessage concurrently (see _state_lock's docstring): two
|
|
327
|
+
# lists appended to from different threads don't necessarily end up in
|
|
328
|
+
# the same relative order.
|
|
314
329
|
tool_calls = state["tool_calls"] or _extract_tool_calls_from_messages(outputs)
|
|
315
|
-
if state["tool_calls"] and len(state["tool_calls"]) == len(state["perf_tool_calls"]):
|
|
316
|
-
# Enrich with the timestamps perf_tool_calls tracked in lockstep,
|
|
317
|
-
# so tool calls interleave correctly in a merged span's timeline.
|
|
318
|
-
tool_calls = [
|
|
319
|
-
{**tc, "start_time": perf.get("start_time"), "end_time": perf.get("end_time")}
|
|
320
|
-
for tc, perf in zip(state["tool_calls"], state["perf_tool_calls"])
|
|
321
|
-
]
|
|
322
330
|
|
|
323
331
|
active_span = self._tracer.current_span
|
|
324
332
|
if active_span is not None:
|
|
@@ -332,6 +340,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
332
340
|
input=state["input"],
|
|
333
341
|
output=output,
|
|
334
342
|
model=state.get("model"),
|
|
343
|
+
framework="langchain",
|
|
335
344
|
input_tokens=state["input_tokens"] or None,
|
|
336
345
|
output_tokens=state["output_tokens"] or None,
|
|
337
346
|
)
|
|
@@ -383,6 +392,7 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
383
392
|
retrieval_steps=state["retrieval_steps"],
|
|
384
393
|
input=state["input"],
|
|
385
394
|
model=state.get("model"),
|
|
395
|
+
framework="langchain",
|
|
386
396
|
input_tokens=state["input_tokens"] or None,
|
|
387
397
|
output_tokens=state["output_tokens"] or None,
|
|
388
398
|
)
|
|
@@ -485,41 +495,42 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
485
495
|
# Extract token usage from LLMResult for this call
|
|
486
496
|
call_input_tokens: Optional[int] = None
|
|
487
497
|
call_output_tokens: Optional[int] = None
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
for
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
498
|
+
with self._state_lock:
|
|
499
|
+
if top and top in self._runs:
|
|
500
|
+
usage = {}
|
|
501
|
+
if hasattr(response, "llm_output") and isinstance(response.llm_output, dict):
|
|
502
|
+
usage = response.llm_output.get("token_usage") or response.llm_output.get("usage") or {}
|
|
503
|
+
# Also check generations for token counts (some providers put it there)
|
|
504
|
+
if not usage and hasattr(response, "generations"):
|
|
505
|
+
for gen_list in (response.generations or []):
|
|
506
|
+
for gen in (gen_list or []):
|
|
507
|
+
gen_info = getattr(gen, "generation_info", None) or {}
|
|
508
|
+
if gen_info.get("prompt_tokens") or gen_info.get("completion_tokens"):
|
|
509
|
+
usage = gen_info
|
|
510
|
+
break
|
|
511
|
+
if usage:
|
|
512
|
+
call_input_tokens = int(
|
|
513
|
+
usage.get("prompt_tokens") or usage.get("input_tokens") or usage.get("prompt_token_count") or 0
|
|
514
|
+
)
|
|
515
|
+
call_output_tokens = int(
|
|
516
|
+
usage.get("completion_tokens") or usage.get("output_tokens") or usage.get("candidates_token_count") or 0
|
|
517
|
+
)
|
|
518
|
+
self._runs[top]["input_tokens"] += call_input_tokens
|
|
519
|
+
self._runs[top]["output_tokens"] += call_output_tokens
|
|
520
|
+
|
|
521
|
+
if start_t is not None and top and top in self._runs:
|
|
522
|
+
steps = self._runs[top]["execution_steps"]
|
|
523
|
+
steps.append({
|
|
524
|
+
"name": f"LLM Call {len(steps) + 1}",
|
|
525
|
+
"duration_ms": (end_t - start_t) * 1000,
|
|
526
|
+
"start_time": start_t,
|
|
527
|
+
"end_time": end_t,
|
|
528
|
+
"model": llm_state.get("model"),
|
|
529
|
+
"input": llm_state.get("input"),
|
|
530
|
+
"output": _extract_llm_output(response),
|
|
531
|
+
"inputTokenSize": call_input_tokens,
|
|
532
|
+
"outputTokenSize": call_output_tokens,
|
|
533
|
+
})
|
|
523
534
|
|
|
524
535
|
# ------------------------------------------------------------------
|
|
525
536
|
# Tool lifecycle
|
|
@@ -561,18 +572,25 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
561
572
|
"input": state["tool_input"],
|
|
562
573
|
"output": str(output),
|
|
563
574
|
"latency_ms": latency_ms,
|
|
575
|
+
"success": True,
|
|
576
|
+
# Set directly on the tool_call dict (not just perf_tool_calls below) so
|
|
577
|
+
# on_chain_end's merged-span path doesn't need to re-pair the two lists by
|
|
578
|
+
# index later. See _state_lock's docstring for why that used to be unsafe.
|
|
579
|
+
"start_time": start_t,
|
|
580
|
+
"end_time": end_t,
|
|
564
581
|
}
|
|
565
582
|
top = self._find_top_ancestor(parent_run_id)
|
|
566
|
-
|
|
567
|
-
self._runs
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
583
|
+
with self._state_lock:
|
|
584
|
+
if top and top in self._runs:
|
|
585
|
+
self._runs[top]["tool_calls"].append(tool_call)
|
|
586
|
+
self._runs[top]["perf_tool_calls"].append({
|
|
587
|
+
"name": state["tool_name"],
|
|
588
|
+
"duration_ms": (end_t - start_t) * 1000,
|
|
589
|
+
"start_time": start_t,
|
|
590
|
+
"end_time": end_t,
|
|
591
|
+
"input": state["tool_input"],
|
|
592
|
+
"output": tool_call["output"],
|
|
593
|
+
})
|
|
576
594
|
|
|
577
595
|
def on_tool_error(
|
|
578
596
|
self,
|
|
@@ -593,18 +611,22 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
593
611
|
"input": state.get("tool_input"),
|
|
594
612
|
"output": f"ERROR: {error}",
|
|
595
613
|
"latency_ms": int((end_t - start_t) * 1000),
|
|
614
|
+
"success": False,
|
|
615
|
+
"start_time": start_t,
|
|
616
|
+
"end_time": end_t,
|
|
596
617
|
}
|
|
597
618
|
top = self._find_top_ancestor(parent_run_id)
|
|
598
|
-
|
|
599
|
-
self._runs
|
|
600
|
-
|
|
601
|
-
"
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
619
|
+
with self._state_lock:
|
|
620
|
+
if top and top in self._runs:
|
|
621
|
+
self._runs[top]["tool_calls"].append(tool_call)
|
|
622
|
+
self._runs[top]["perf_tool_calls"].append({
|
|
623
|
+
"name": state.get("tool_name", "unknown"),
|
|
624
|
+
"duration_ms": (end_t - start_t) * 1000,
|
|
625
|
+
"start_time": start_t,
|
|
626
|
+
"end_time": end_t,
|
|
627
|
+
"input": tool_call["input"],
|
|
628
|
+
"output": tool_call["output"],
|
|
629
|
+
})
|
|
608
630
|
|
|
609
631
|
# ------------------------------------------------------------------
|
|
610
632
|
# Retriever lifecycle
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from agentx.monitor.client import MonitorClient
|
|
2
|
-
from agentx.monitor.models import MonitorPattern, MonitorSignal, SignalOccurrence
|
|
2
|
+
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal, SignalOccurrence
|
|
3
3
|
from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
|
|
4
|
+
from agentx.monitor.profile import MonitorProfileClient
|
|
4
5
|
from agentx.monitor.signals import MonitorSignalClient
|
|
5
6
|
|
|
6
7
|
__all__ = [
|
|
@@ -8,6 +9,8 @@ __all__ = [
|
|
|
8
9
|
"MonitorPattern",
|
|
9
10
|
"MonitorPatternBuilder",
|
|
10
11
|
"MonitorPatternClient",
|
|
12
|
+
"MonitorProfile",
|
|
13
|
+
"MonitorProfileClient",
|
|
11
14
|
"MonitorSignal",
|
|
12
15
|
"SignalOccurrence",
|
|
13
16
|
"MonitorSignalClient",
|
|
@@ -7,7 +7,7 @@ from typing import Any, List, Optional
|
|
|
7
7
|
|
|
8
8
|
import requests
|
|
9
9
|
|
|
10
|
-
from agentx.monitor.models import MonitorPattern, MonitorSignal
|
|
10
|
+
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
@@ -70,9 +70,11 @@ class MonitorClient:
|
|
|
70
70
|
|
|
71
71
|
from agentx.monitor.patterns import MonitorPatternClient
|
|
72
72
|
from agentx.monitor.signals import MonitorSignalClient
|
|
73
|
+
from agentx.monitor.profile import MonitorProfileClient
|
|
73
74
|
|
|
74
75
|
self.patterns = MonitorPatternClient(self)
|
|
75
76
|
self.signals = MonitorSignalClient(self)
|
|
77
|
+
self.profile = MonitorProfileClient(self)
|
|
76
78
|
|
|
77
79
|
# ------------------------------------------------------------------
|
|
78
80
|
# Low-level HTTP
|
|
@@ -166,3 +168,20 @@ class MonitorClient:
|
|
|
166
168
|
"GET", f"/signals/{signal_id}", params=self._workspace_params()
|
|
167
169
|
)
|
|
168
170
|
return MonitorSignal(**data["signal"])
|
|
171
|
+
|
|
172
|
+
# ------------------------------------------------------------------
|
|
173
|
+
# Profile endpoints
|
|
174
|
+
# ------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
def get_profile(self, agent_id: str) -> Optional[MonitorProfile]:
|
|
177
|
+
data = self._request(
|
|
178
|
+
"GET", f"/profiles/{agent_id}", params=self._workspace_params()
|
|
179
|
+
)
|
|
180
|
+
profile = data.get("profile")
|
|
181
|
+
return MonitorProfile(**profile) if profile else None
|
|
182
|
+
|
|
183
|
+
def update_profile(self, agent_id: str, payload: dict) -> MonitorProfile:
|
|
184
|
+
data = self._request(
|
|
185
|
+
"PUT", f"/profiles/{agent_id}", json=self._with_workspace(payload)
|
|
186
|
+
)
|
|
187
|
+
return MonitorProfile(**data["profile"])
|
|
@@ -51,6 +51,36 @@ class SignalOccurrence(BaseModel):
|
|
|
51
51
|
extra = "ignore"
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
class MonitorProfile(BaseModel):
|
|
55
|
+
"""A single agent's Monitor coverage/detection settings: what gets sampled, how long it's
|
|
56
|
+
kept, whether info (clean-run) signals are logged, per-check threshold overrides (e.g.
|
|
57
|
+
``threshold_overrides["latencyMs"]`` for the built-in "Latency regression" pattern), and the
|
|
58
|
+
approval policy gating autotune actions. See ``client.monitor.profile.get()/update()``.
|
|
59
|
+
``None`` from ``get()`` means this agent has never been configured and is running on
|
|
60
|
+
platform defaults (e.g. the built-in latency threshold defaults to 20000ms).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
id: Optional[str] = Field(default=None, alias="_id")
|
|
64
|
+
workspace_id: Optional[str] = Field(default=None, alias="workspaceId")
|
|
65
|
+
agent_id: Optional[Any] = Field(default=None, alias="agentId")
|
|
66
|
+
enabled: bool = True
|
|
67
|
+
failure_detection_enabled: bool = Field(default=True, alias="failureDetectionEnabled")
|
|
68
|
+
info_detection_enabled: bool = Field(default=True, alias="infoDetectionEnabled")
|
|
69
|
+
coverage_mode: str = Field(default="all", alias="coverageMode")
|
|
70
|
+
sample_rate: float = Field(default=0.1, alias="sampleRate")
|
|
71
|
+
channels: List[str] = Field(default_factory=list)
|
|
72
|
+
threshold_overrides: Optional[Dict[str, Any]] = Field(default=None, alias="thresholdOverrides")
|
|
73
|
+
retention_days: int = Field(default=30, alias="retentionDays")
|
|
74
|
+
redaction_mode: str = Field(default="standard", alias="redactionMode")
|
|
75
|
+
approval_policy: Optional[Dict[str, str]] = Field(default=None, alias="approvalPolicy")
|
|
76
|
+
created_at: Optional[str] = Field(default=None, alias="createdAt")
|
|
77
|
+
updated_at: Optional[str] = Field(default=None, alias="updatedAt")
|
|
78
|
+
|
|
79
|
+
class Config:
|
|
80
|
+
populate_by_name = True
|
|
81
|
+
extra = "ignore"
|
|
82
|
+
|
|
83
|
+
|
|
54
84
|
class MonitorSignal(BaseModel):
|
|
55
85
|
"""An alert/finding produced when a trace matched a pattern (or, for a "proper" pattern,
|
|
56
86
|
a healthy tally). Read-only from the SDK, see ``client.monitor.signals.list()/get()``, a
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from agentx.monitor.models import MonitorProfile
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from agentx.monitor.client import MonitorClient
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MonitorProfileClient:
|
|
15
|
+
"""Thin wrapper surfaced as ``client.monitor.profile``: get/update a single agent's Monitor
|
|
16
|
+
coverage and detection settings (coverage mode, sample rate, retention, redaction, approval
|
|
17
|
+
policy, and threshold_overrides, e.g. the built-in "Latency regression" pattern's threshold).
|
|
18
|
+
|
|
19
|
+
Unlike patterns/signals, a profile is scoped to one agent per call, since that mirrors how
|
|
20
|
+
the dashboard's per-agent monitoring settings dialog works: ``get(agent_id)`` returns ``None``
|
|
21
|
+
for an agent that's never been configured (still on platform defaults), and ``update(agent_id,
|
|
22
|
+
...)`` upserts one.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, client: "MonitorClient"):
|
|
26
|
+
self._client = client
|
|
27
|
+
|
|
28
|
+
def get(self, agent_id: str) -> Optional[MonitorProfile]:
|
|
29
|
+
return self._client.get_profile(agent_id)
|
|
30
|
+
|
|
31
|
+
def update(
|
|
32
|
+
self,
|
|
33
|
+
agent_id: str,
|
|
34
|
+
*,
|
|
35
|
+
enabled: Optional[bool] = None,
|
|
36
|
+
failure_detection_enabled: Optional[bool] = None,
|
|
37
|
+
info_detection_enabled: Optional[bool] = None,
|
|
38
|
+
coverage_mode: Optional[str] = None,
|
|
39
|
+
sample_rate: Optional[float] = None,
|
|
40
|
+
channels: Optional[List[str]] = None,
|
|
41
|
+
dataset_id: Optional[str] = None,
|
|
42
|
+
threshold_overrides: Optional[Dict[str, Any]] = None,
|
|
43
|
+
retention_days: Optional[int] = None,
|
|
44
|
+
redaction_mode: Optional[str] = None,
|
|
45
|
+
approval_policy: Optional[Dict[str, str]] = None,
|
|
46
|
+
) -> MonitorProfile:
|
|
47
|
+
"""Update (and enable, if not already) this agent's Monitor profile. Only fields passed
|
|
48
|
+
here are changed; everything else on the existing profile is left as is.
|
|
49
|
+
|
|
50
|
+
Example, overriding the built-in "Latency regression" pattern's threshold for one agent::
|
|
51
|
+
|
|
52
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
53
|
+
"""
|
|
54
|
+
payload: Dict[str, Any] = {
|
|
55
|
+
"enabled": enabled,
|
|
56
|
+
"failureDetectionEnabled": failure_detection_enabled,
|
|
57
|
+
"infoDetectionEnabled": info_detection_enabled,
|
|
58
|
+
"coverageMode": coverage_mode,
|
|
59
|
+
"sampleRate": sample_rate,
|
|
60
|
+
"channels": channels,
|
|
61
|
+
"datasetId": dataset_id,
|
|
62
|
+
"thresholdOverrides": threshold_overrides,
|
|
63
|
+
"retentionDays": retention_days,
|
|
64
|
+
"redactionMode": redaction_mode,
|
|
65
|
+
"approvalPolicy": approval_policy,
|
|
66
|
+
}
|
|
67
|
+
payload = {k: v for k, v in payload.items() if v is not None}
|
|
68
|
+
return self._client.update_profile(agent_id, payload)
|
|
@@ -114,6 +114,9 @@ class _TraceSpan:
|
|
|
114
114
|
self._execution_steps: list = []
|
|
115
115
|
self._retrieval_steps: list = []
|
|
116
116
|
self._captured_model: Optional[str] = None
|
|
117
|
+
# Adopted from a merged child run (e.g. AgentXCallbackHandler) when this span itself
|
|
118
|
+
# wasn't opened with an explicit framework= — see _merge_child_run below.
|
|
119
|
+
self._captured_framework: Optional[str] = None
|
|
117
120
|
self._input_tokens: int = 0
|
|
118
121
|
self._output_tokens: int = 0
|
|
119
122
|
# Guards _merge_child_run — with Tracer.use_span(), multiple threads
|
|
@@ -186,7 +189,7 @@ class _TraceSpan:
|
|
|
186
189
|
latency_ms=latency_ms,
|
|
187
190
|
error=self._error,
|
|
188
191
|
metadata=self._metadata,
|
|
189
|
-
framework=self._framework,
|
|
192
|
+
framework=self._framework or self._captured_framework,
|
|
190
193
|
model=self._model or self._captured_model,
|
|
191
194
|
tool_calls=self.tool_calls or None,
|
|
192
195
|
session_id=self._session_id,
|
|
@@ -250,6 +253,7 @@ class _TraceSpan:
|
|
|
250
253
|
input: Any = None,
|
|
251
254
|
output: Any = None,
|
|
252
255
|
model: Optional[str] = None,
|
|
256
|
+
framework: Optional[str] = None,
|
|
253
257
|
input_tokens: Optional[int] = None,
|
|
254
258
|
output_tokens: Optional[int] = None,
|
|
255
259
|
) -> None:
|
|
@@ -281,6 +285,8 @@ class _TraceSpan:
|
|
|
281
285
|
self.output = output
|
|
282
286
|
if model and not self._captured_model:
|
|
283
287
|
self._captured_model = model
|
|
288
|
+
if framework and not self._captured_framework:
|
|
289
|
+
self._captured_framework = framework
|
|
284
290
|
if input_tokens:
|
|
285
291
|
self._input_tokens += input_tokens
|
|
286
292
|
if output_tokens:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = "0.6.10"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.10
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -72,6 +72,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
73
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
74
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
75
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
75
76
|
- [Links](#links)
|
|
76
77
|
|
|
77
78
|
---
|
|
@@ -273,6 +274,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
273
274
|
print(signal.summary, signal.occurrence_count)
|
|
274
275
|
```
|
|
275
276
|
|
|
277
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
281
|
+
```
|
|
282
|
+
|
|
276
283
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
277
284
|
|
|
278
285
|
---
|
|
@@ -306,6 +313,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
306
313
|
|
|
307
314
|
---
|
|
308
315
|
|
|
316
|
+
## Self-host
|
|
317
|
+
|
|
318
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
agentx-trace-eval --dev
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
328
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
309
335
|
## Links
|
|
310
336
|
|
|
311
337
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -314,3 +340,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
314
340
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
315
341
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
316
342
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
343
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -3,6 +3,7 @@ README.md
|
|
|
3
3
|
setup.py
|
|
4
4
|
agentx/__init__.py
|
|
5
5
|
agentx/agentx.py
|
|
6
|
+
agentx/cli.py
|
|
6
7
|
agentx/exceptions.py
|
|
7
8
|
agentx/py.typed
|
|
8
9
|
agentx/util.py
|
|
@@ -34,6 +35,7 @@ agentx/monitor/__init__.py
|
|
|
34
35
|
agentx/monitor/client.py
|
|
35
36
|
agentx/monitor/models.py
|
|
36
37
|
agentx/monitor/patterns.py
|
|
38
|
+
agentx/monitor/profile.py
|
|
37
39
|
agentx/monitor/signals.py
|
|
38
40
|
agentx/resources/__init__.py
|
|
39
41
|
agentx/resources/agent.py
|
|
@@ -46,6 +48,7 @@ agentx/tracing/tracer.py
|
|
|
46
48
|
agentx_python.egg-info/PKG-INFO
|
|
47
49
|
agentx_python.egg-info/SOURCES.txt
|
|
48
50
|
agentx_python.egg-info/dependency_links.txt
|
|
51
|
+
agentx_python.egg-info/entry_points.txt
|
|
49
52
|
agentx_python.egg-info/not-zip-safe
|
|
50
53
|
agentx_python.egg-info/requires.txt
|
|
51
54
|
agentx_python.egg-info/top_level.txt
|
|
@@ -26,6 +26,12 @@ setup(
|
|
|
26
26
|
# checking for anything that touches the SDK.
|
|
27
27
|
package_data={"agentx": ["py.typed"]},
|
|
28
28
|
zip_safe=False,
|
|
29
|
+
entry_points={
|
|
30
|
+
# Thin launcher for the self-hostable governance engine (AgentX-ai/AgentX-trace-eval),
|
|
31
|
+
# not this SDK itself, see agentx/cli.py's module docstring for why it lazily downloads
|
|
32
|
+
# rather than bundling that engine in this package.
|
|
33
|
+
"console_scripts": ["agentx-trace-eval=agentx.cli:main"],
|
|
34
|
+
},
|
|
29
35
|
install_requires=[
|
|
30
36
|
"urllib3>=1.26.11",
|
|
31
37
|
"certifi",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
VERSION = "0.6.8"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|