agentx-python 0.6.9__tar.gz → 0.6.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.9/agentx_python.egg-info → agentx_python-0.6.10}/PKG-INFO +28 -1
- {agentx_python-0.6.9 → agentx_python-0.6.10}/README.md +27 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/agentx.py +3 -1
- agentx_python-0.6.10/agentx/cli.py +158 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/monitor/__init__.py +4 -1
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/monitor/client.py +20 -1
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/monitor/models.py +30 -0
- agentx_python-0.6.10/agentx/monitor/profile.py +68 -0
- agentx_python-0.6.10/agentx/version.py +1 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10/agentx_python.egg-info}/PKG-INFO +28 -1
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx_python.egg-info/SOURCES.txt +3 -0
- agentx_python-0.6.10/agentx_python.egg-info/entry_points.txt +2 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/setup.py +6 -0
- agentx_python-0.6.9/agentx/version.py +0 -1
- {agentx_python-0.6.9 → agentx_python-0.6.10}/LICENSE +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/client.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/models.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/runner.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/exceptions.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/_perf.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/crewai.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/google_adk.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/langchain.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/integrations/openai_agents.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/py.typed +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/tracing/tracer.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx/util.py +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/setup.cfg +0 -0
- {agentx_python-0.6.9 → agentx_python-0.6.10}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.10
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -72,6 +72,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
73
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
74
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
75
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
75
76
|
- [Links](#links)
|
|
76
77
|
|
|
77
78
|
---
|
|
@@ -273,6 +274,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
273
274
|
print(signal.summary, signal.occurrence_count)
|
|
274
275
|
```
|
|
275
276
|
|
|
277
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
281
|
+
```
|
|
282
|
+
|
|
276
283
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
277
284
|
|
|
278
285
|
---
|
|
@@ -306,6 +313,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
306
313
|
|
|
307
314
|
---
|
|
308
315
|
|
|
316
|
+
## Self-host
|
|
317
|
+
|
|
318
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
agentx-trace-eval --dev
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
328
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
309
335
|
## Links
|
|
310
336
|
|
|
311
337
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -314,3 +340,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
314
340
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
315
341
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
316
342
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
343
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -24,6 +24,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
24
24
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
25
25
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
26
26
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
27
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
27
28
|
- [Links](#links)
|
|
28
29
|
|
|
29
30
|
---
|
|
@@ -225,6 +226,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
225
226
|
print(signal.summary, signal.occurrence_count)
|
|
226
227
|
```
|
|
227
228
|
|
|
229
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
233
|
+
```
|
|
234
|
+
|
|
228
235
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
229
236
|
|
|
230
237
|
---
|
|
@@ -258,6 +265,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
258
265
|
|
|
259
266
|
---
|
|
260
267
|
|
|
268
|
+
## Self-host
|
|
269
|
+
|
|
270
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
agentx-trace-eval --dev
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
280
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
284
|
+
|
|
285
|
+
---
|
|
286
|
+
|
|
261
287
|
## Links
|
|
262
288
|
|
|
263
289
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -266,3 +292,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
266
292
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
267
293
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
268
294
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
295
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -44,7 +44,9 @@ class AgentX:
|
|
|
44
44
|
|
|
45
45
|
# Monitor: create/reuse patterns (client.monitor.patterns) that a trace can be checked
|
|
46
46
|
# against at send time via tracer.trace(..., monitor=True, pattern_ids=[...]), then read
|
|
47
|
-
# back the resulting alerts/findings with client.monitor.signals.
|
|
47
|
+
# back the resulting alerts/findings with client.monitor.signals. Per-agent coverage and
|
|
48
|
+
# detection settings (sample rate, retention, threshold overrides like the built-in
|
|
49
|
+
# "Latency regression" pattern's threshold) are client.monitor.profile.
|
|
48
50
|
self.monitor = MonitorClient(
|
|
49
51
|
api_key=self.api_key,
|
|
50
52
|
sdk_version=VERSION,
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""
|
|
2
|
+
`agentx-trace-eval` — thin launcher for AgentX's self-hostable governance engine (Trace,
|
|
3
|
+
Evaluate, Monitor), published separately at github.com/AgentX-ai/AgentX-trace-eval (a Go CLI
|
|
4
|
+
wrapping a Bun-compiled TypeScript engine, not Python). That compiled engine binary is tens of
|
|
5
|
+
megabytes; most `pip install agentx-python` installs are just this SDK talking to the hosted
|
|
6
|
+
AgentX SaaS and would never touch it, so it isn't bundled in this package. Instead, this command
|
|
7
|
+
downloads the matching release into ~/.agentx/bin the first time it's needed (mirroring
|
|
8
|
+
AgentX-trace-eval's own install.sh) and then hands off to the real `agentx-server` binary.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
agentx-trace-eval --dev
|
|
12
|
+
agentx-trace-eval --port 5000 --db-url postgres://...
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import platform
|
|
17
|
+
import shutil
|
|
18
|
+
import stat
|
|
19
|
+
import sys
|
|
20
|
+
import tarfile
|
|
21
|
+
import tempfile
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Tuple
|
|
24
|
+
|
|
25
|
+
import requests
|
|
26
|
+
|
|
27
|
+
REPO = "AgentX-ai/AgentX-trace-eval"
|
|
28
|
+
INSTALL_DIR = Path(os.environ.get("AGENTX_INSTALL_DIR", str(Path.home() / ".agentx" / "bin")))
|
|
29
|
+
_BIN_NAMES = ("agentx", "agentx-server", "agentx-engine")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _platform_tag() -> Tuple[str, str]:
|
|
33
|
+
system = platform.system()
|
|
34
|
+
if system == "Darwin":
|
|
35
|
+
os_name = "darwin"
|
|
36
|
+
elif system == "Linux":
|
|
37
|
+
os_name = "linux"
|
|
38
|
+
else:
|
|
39
|
+
raise SystemExit(
|
|
40
|
+
f"agentx-trace-eval: unsupported OS {system!r} (self-host currently supports macOS and Linux)"
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
machine = platform.machine().lower()
|
|
44
|
+
if machine in ("arm64", "aarch64"):
|
|
45
|
+
arch = "arm64"
|
|
46
|
+
elif machine in ("x86_64", "amd64"):
|
|
47
|
+
arch = "amd64"
|
|
48
|
+
else:
|
|
49
|
+
raise SystemExit(f"agentx-trace-eval: unsupported architecture {machine!r}")
|
|
50
|
+
|
|
51
|
+
return os_name, arch
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _release_url(asset: str, version: str) -> str:
|
|
55
|
+
if version == "latest":
|
|
56
|
+
return f"https://github.com/{REPO}/releases/latest/download/{asset}"
|
|
57
|
+
return f"https://github.com/{REPO}/releases/download/{version}/{asset}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _download(url: str, dest: Path) -> None:
|
|
61
|
+
response = requests.get(url, stream=True, timeout=60)
|
|
62
|
+
response.raise_for_status()
|
|
63
|
+
with open(dest, "wb") as f:
|
|
64
|
+
for chunk in response.iter_content(chunk_size=1 << 16):
|
|
65
|
+
f.write(chunk)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _extract_tar(archive: Path, dest_dir: Path) -> None:
|
|
69
|
+
with tarfile.open(archive) as tar:
|
|
70
|
+
try:
|
|
71
|
+
# filter="data" (PEP 706, Python 3.12+) rejects absolute paths/symlink escapes.
|
|
72
|
+
# Belt-and-suspenders here since these are trusted release assets built by our own CI
|
|
73
|
+
# (see REPO above), not arbitrary user-supplied archives.
|
|
74
|
+
tar.extractall(dest_dir, filter="data")
|
|
75
|
+
except TypeError:
|
|
76
|
+
tar.extractall(dest_dir) # Python < 3.12: filter kwarg doesn't exist yet
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _install(version: str = "latest") -> None:
|
|
80
|
+
os_name, arch = _platform_tag()
|
|
81
|
+
INSTALL_DIR.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
|
|
83
|
+
print(f"agentx-trace-eval: downloading agentx ({os_name}/{arch})...", file=sys.stderr)
|
|
84
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
85
|
+
tmp_dir = Path(tmp)
|
|
86
|
+
archive = tmp_dir / "agentx.tar.gz"
|
|
87
|
+
url = _release_url(f"agentx_{os_name}_{arch}.tar.gz", version)
|
|
88
|
+
try:
|
|
89
|
+
_download(url, archive)
|
|
90
|
+
except requests.HTTPError as exc:
|
|
91
|
+
raise SystemExit(
|
|
92
|
+
f"agentx-trace-eval: failed to download {url} ({exc}).\n"
|
|
93
|
+
"No published release found; see AgentX-trace-eval's README for building from source."
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
_extract_tar(archive, tmp_dir)
|
|
97
|
+
|
|
98
|
+
found_any = False
|
|
99
|
+
for name in _BIN_NAMES:
|
|
100
|
+
src = tmp_dir / name
|
|
101
|
+
if not src.exists():
|
|
102
|
+
continue
|
|
103
|
+
found_any = True
|
|
104
|
+
dest = INSTALL_DIR / name
|
|
105
|
+
shutil.move(str(src), str(dest))
|
|
106
|
+
dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
|
|
107
|
+
|
|
108
|
+
if not found_any:
|
|
109
|
+
raise SystemExit(f"agentx-trace-eval: downloaded archive from {url} didn't contain any of {_BIN_NAMES}")
|
|
110
|
+
|
|
111
|
+
if os.environ.get("AGENTX_TRACE_EVAL_SKIP_WEB"):
|
|
112
|
+
return
|
|
113
|
+
|
|
114
|
+
# Best-effort: the dashboard is a separate, platform-independent asset (see
|
|
115
|
+
# AgentX-trace-eval's README's "Dashboard release"). Missing it shouldn't block getting the
|
|
116
|
+
# engine running headless, so a failure here warns and continues rather than raising.
|
|
117
|
+
print("agentx-trace-eval: downloading dashboard...", file=sys.stderr)
|
|
118
|
+
web_dir = INSTALL_DIR / "web"
|
|
119
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
120
|
+
web_archive = Path(tmp) / "agentx-web.tar.gz"
|
|
121
|
+
web_url = _release_url("agentx-web.tar.gz", version)
|
|
122
|
+
try:
|
|
123
|
+
_download(web_url, web_archive)
|
|
124
|
+
except requests.HTTPError:
|
|
125
|
+
print(
|
|
126
|
+
f"agentx-trace-eval: no dashboard bundle found at {web_url}, continuing without one",
|
|
127
|
+
file=sys.stderr,
|
|
128
|
+
)
|
|
129
|
+
return
|
|
130
|
+
if web_dir.exists():
|
|
131
|
+
shutil.rmtree(web_dir)
|
|
132
|
+
web_dir.mkdir(parents=True)
|
|
133
|
+
_extract_tar(web_archive, web_dir)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def ensure_installed(version: str = "latest") -> Path:
|
|
137
|
+
"""Downloads agentx-server (+ its engine) into ~/.agentx/bin if not already present there.
|
|
138
|
+
Returns the path to the agentx-server executable. Set AGENTX_INSTALL_DIR to change where
|
|
139
|
+
this looks/installs; set AGENTX_TRACE_EVAL_VERSION to pin a release tag instead of latest."""
|
|
140
|
+
server_path = INSTALL_DIR / "agentx-server"
|
|
141
|
+
if not server_path.exists():
|
|
142
|
+
_install(version=version)
|
|
143
|
+
return server_path
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def main() -> None:
|
|
147
|
+
version = os.environ.get("AGENTX_TRACE_EVAL_VERSION", "latest")
|
|
148
|
+
server_path = ensure_installed(version=version)
|
|
149
|
+
if not server_path.exists():
|
|
150
|
+
raise SystemExit(f"agentx-trace-eval: {server_path} still missing after install, giving up")
|
|
151
|
+
|
|
152
|
+
# os.execv replaces this process rather than spawning a subprocess: signals, stdio, and the
|
|
153
|
+
# exit code all pass straight through to agentx-server, same as invoking it directly.
|
|
154
|
+
os.execv(str(server_path), [str(server_path)] + sys.argv[1:])
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
main()
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from agentx.monitor.client import MonitorClient
|
|
2
|
-
from agentx.monitor.models import MonitorPattern, MonitorSignal, SignalOccurrence
|
|
2
|
+
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal, SignalOccurrence
|
|
3
3
|
from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
|
|
4
|
+
from agentx.monitor.profile import MonitorProfileClient
|
|
4
5
|
from agentx.monitor.signals import MonitorSignalClient
|
|
5
6
|
|
|
6
7
|
__all__ = [
|
|
@@ -8,6 +9,8 @@ __all__ = [
|
|
|
8
9
|
"MonitorPattern",
|
|
9
10
|
"MonitorPatternBuilder",
|
|
10
11
|
"MonitorPatternClient",
|
|
12
|
+
"MonitorProfile",
|
|
13
|
+
"MonitorProfileClient",
|
|
11
14
|
"MonitorSignal",
|
|
12
15
|
"SignalOccurrence",
|
|
13
16
|
"MonitorSignalClient",
|
|
@@ -7,7 +7,7 @@ from typing import Any, List, Optional
|
|
|
7
7
|
|
|
8
8
|
import requests
|
|
9
9
|
|
|
10
|
-
from agentx.monitor.models import MonitorPattern, MonitorSignal
|
|
10
|
+
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal
|
|
11
11
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
@@ -70,9 +70,11 @@ class MonitorClient:
|
|
|
70
70
|
|
|
71
71
|
from agentx.monitor.patterns import MonitorPatternClient
|
|
72
72
|
from agentx.monitor.signals import MonitorSignalClient
|
|
73
|
+
from agentx.monitor.profile import MonitorProfileClient
|
|
73
74
|
|
|
74
75
|
self.patterns = MonitorPatternClient(self)
|
|
75
76
|
self.signals = MonitorSignalClient(self)
|
|
77
|
+
self.profile = MonitorProfileClient(self)
|
|
76
78
|
|
|
77
79
|
# ------------------------------------------------------------------
|
|
78
80
|
# Low-level HTTP
|
|
@@ -166,3 +168,20 @@ class MonitorClient:
|
|
|
166
168
|
"GET", f"/signals/{signal_id}", params=self._workspace_params()
|
|
167
169
|
)
|
|
168
170
|
return MonitorSignal(**data["signal"])
|
|
171
|
+
|
|
172
|
+
# ------------------------------------------------------------------
|
|
173
|
+
# Profile endpoints
|
|
174
|
+
# ------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
def get_profile(self, agent_id: str) -> Optional[MonitorProfile]:
|
|
177
|
+
data = self._request(
|
|
178
|
+
"GET", f"/profiles/{agent_id}", params=self._workspace_params()
|
|
179
|
+
)
|
|
180
|
+
profile = data.get("profile")
|
|
181
|
+
return MonitorProfile(**profile) if profile else None
|
|
182
|
+
|
|
183
|
+
def update_profile(self, agent_id: str, payload: dict) -> MonitorProfile:
|
|
184
|
+
data = self._request(
|
|
185
|
+
"PUT", f"/profiles/{agent_id}", json=self._with_workspace(payload)
|
|
186
|
+
)
|
|
187
|
+
return MonitorProfile(**data["profile"])
|
|
@@ -51,6 +51,36 @@ class SignalOccurrence(BaseModel):
|
|
|
51
51
|
extra = "ignore"
|
|
52
52
|
|
|
53
53
|
|
|
54
|
+
class MonitorProfile(BaseModel):
|
|
55
|
+
"""A single agent's Monitor coverage/detection settings: what gets sampled, how long it's
|
|
56
|
+
kept, whether info (clean-run) signals are logged, per-check threshold overrides (e.g.
|
|
57
|
+
``threshold_overrides["latencyMs"]`` for the built-in "Latency regression" pattern), and the
|
|
58
|
+
approval policy gating autotune actions. See ``client.monitor.profile.get()/update()``.
|
|
59
|
+
``None`` from ``get()`` means this agent has never been configured and is running on
|
|
60
|
+
platform defaults (e.g. the built-in latency threshold defaults to 20000ms).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
id: Optional[str] = Field(default=None, alias="_id")
|
|
64
|
+
workspace_id: Optional[str] = Field(default=None, alias="workspaceId")
|
|
65
|
+
agent_id: Optional[Any] = Field(default=None, alias="agentId")
|
|
66
|
+
enabled: bool = True
|
|
67
|
+
failure_detection_enabled: bool = Field(default=True, alias="failureDetectionEnabled")
|
|
68
|
+
info_detection_enabled: bool = Field(default=True, alias="infoDetectionEnabled")
|
|
69
|
+
coverage_mode: str = Field(default="all", alias="coverageMode")
|
|
70
|
+
sample_rate: float = Field(default=0.1, alias="sampleRate")
|
|
71
|
+
channels: List[str] = Field(default_factory=list)
|
|
72
|
+
threshold_overrides: Optional[Dict[str, Any]] = Field(default=None, alias="thresholdOverrides")
|
|
73
|
+
retention_days: int = Field(default=30, alias="retentionDays")
|
|
74
|
+
redaction_mode: str = Field(default="standard", alias="redactionMode")
|
|
75
|
+
approval_policy: Optional[Dict[str, str]] = Field(default=None, alias="approvalPolicy")
|
|
76
|
+
created_at: Optional[str] = Field(default=None, alias="createdAt")
|
|
77
|
+
updated_at: Optional[str] = Field(default=None, alias="updatedAt")
|
|
78
|
+
|
|
79
|
+
class Config:
|
|
80
|
+
populate_by_name = True
|
|
81
|
+
extra = "ignore"
|
|
82
|
+
|
|
83
|
+
|
|
54
84
|
class MonitorSignal(BaseModel):
|
|
55
85
|
"""An alert/finding produced when a trace matched a pattern (or, for a "proper" pattern,
|
|
56
86
|
a healthy tally). Read-only from the SDK, see ``client.monitor.signals.list()/get()``, a
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from agentx.monitor.models import MonitorProfile
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from agentx.monitor.client import MonitorClient
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MonitorProfileClient:
|
|
15
|
+
"""Thin wrapper surfaced as ``client.monitor.profile``: get/update a single agent's Monitor
|
|
16
|
+
coverage and detection settings (coverage mode, sample rate, retention, redaction, approval
|
|
17
|
+
policy, and threshold_overrides, e.g. the built-in "Latency regression" pattern's threshold).
|
|
18
|
+
|
|
19
|
+
Unlike patterns/signals, a profile is scoped to one agent per call, since that mirrors how
|
|
20
|
+
the dashboard's per-agent monitoring settings dialog works: ``get(agent_id)`` returns ``None``
|
|
21
|
+
for an agent that's never been configured (still on platform defaults), and ``update(agent_id,
|
|
22
|
+
...)`` upserts one.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
def __init__(self, client: "MonitorClient"):
|
|
26
|
+
self._client = client
|
|
27
|
+
|
|
28
|
+
def get(self, agent_id: str) -> Optional[MonitorProfile]:
|
|
29
|
+
return self._client.get_profile(agent_id)
|
|
30
|
+
|
|
31
|
+
def update(
|
|
32
|
+
self,
|
|
33
|
+
agent_id: str,
|
|
34
|
+
*,
|
|
35
|
+
enabled: Optional[bool] = None,
|
|
36
|
+
failure_detection_enabled: Optional[bool] = None,
|
|
37
|
+
info_detection_enabled: Optional[bool] = None,
|
|
38
|
+
coverage_mode: Optional[str] = None,
|
|
39
|
+
sample_rate: Optional[float] = None,
|
|
40
|
+
channels: Optional[List[str]] = None,
|
|
41
|
+
dataset_id: Optional[str] = None,
|
|
42
|
+
threshold_overrides: Optional[Dict[str, Any]] = None,
|
|
43
|
+
retention_days: Optional[int] = None,
|
|
44
|
+
redaction_mode: Optional[str] = None,
|
|
45
|
+
approval_policy: Optional[Dict[str, str]] = None,
|
|
46
|
+
) -> MonitorProfile:
|
|
47
|
+
"""Update (and enable, if not already) this agent's Monitor profile. Only fields passed
|
|
48
|
+
here are changed; everything else on the existing profile is left as is.
|
|
49
|
+
|
|
50
|
+
Example, overriding the built-in "Latency regression" pattern's threshold for one agent::
|
|
51
|
+
|
|
52
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
53
|
+
"""
|
|
54
|
+
payload: Dict[str, Any] = {
|
|
55
|
+
"enabled": enabled,
|
|
56
|
+
"failureDetectionEnabled": failure_detection_enabled,
|
|
57
|
+
"infoDetectionEnabled": info_detection_enabled,
|
|
58
|
+
"coverageMode": coverage_mode,
|
|
59
|
+
"sampleRate": sample_rate,
|
|
60
|
+
"channels": channels,
|
|
61
|
+
"datasetId": dataset_id,
|
|
62
|
+
"thresholdOverrides": threshold_overrides,
|
|
63
|
+
"retentionDays": retention_days,
|
|
64
|
+
"redactionMode": redaction_mode,
|
|
65
|
+
"approvalPolicy": approval_policy,
|
|
66
|
+
}
|
|
67
|
+
payload = {k: v for k, v in payload.items() if v is not None}
|
|
68
|
+
return self._client.update_profile(agent_id, payload)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = "0.6.10"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.10
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -72,6 +72,7 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
|
|
|
72
72
|
- [Production tracing](#production-tracing) — record live agent runs from any framework
|
|
73
73
|
- [Monitor](#monitor) — automatic production monitoring, patterns and signals
|
|
74
74
|
- [Custom agent evaluations](#custom-agent-evaluations) — LLM-as-a-judge, cosine / Jaccard similarity
|
|
75
|
+
- [Self-host](#self-host) — run Trace/Evaluate/Monitor on your own machine instead of the hosted dashboard
|
|
75
76
|
- [Links](#links)
|
|
76
77
|
|
|
77
78
|
---
|
|
@@ -273,6 +274,12 @@ for signal in client.monitor.signals.list(severity="high"):
|
|
|
273
274
|
print(signal.summary, signal.occurrence_count)
|
|
274
275
|
```
|
|
275
276
|
|
|
277
|
+
Per-agent coverage/threshold settings (sample rate, retention, and threshold overrides like the built-in "Latency regression" pattern's threshold) are `client.monitor.profile.get()`/`.update()`:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
281
|
+
```
|
|
282
|
+
|
|
276
283
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
277
284
|
|
|
278
285
|
---
|
|
@@ -306,6 +313,25 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
306
313
|
|
|
307
314
|
---
|
|
308
315
|
|
|
316
|
+
## Self-host
|
|
317
|
+
|
|
318
|
+
Prefer to run Trace/Evaluate/Monitor locally instead of the hosted dashboard — no account, bring your own LLM keys? This SDK ships a launcher for [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval), a separate, portable governance engine:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
agentx-trace-eval --dev
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The first run downloads the engine (and dashboard) into `~/.agentx/bin` and prints a local API key; every run after that just starts it. Point this SDK at it instead of the hosted API:
|
|
325
|
+
|
|
326
|
+
```bash
|
|
327
|
+
export AGENTX_API_BASE_URL=http://localhost:4700/api/v1
|
|
328
|
+
export AGENTX_API_KEY=<printed by agentx-trace-eval on first run>
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
`agentx-trace-eval` isn't this SDK's own code — the engine itself is a separate, compiled binary, downloaded on demand rather than bundled into this package, so installing `agentx-python` doesn't get any heavier for the (much more common) case of just talking to the hosted AgentX API. See that repo's README for what's included, and `AGENTX_INSTALL_DIR`/`AGENTX_TRACE_EVAL_VERSION`/`AGENTX_TRACE_EVAL_SKIP_WEB` env vars to control where/what it installs.
|
|
332
|
+
|
|
333
|
+
---
|
|
334
|
+
|
|
309
335
|
## Links
|
|
310
336
|
|
|
311
337
|
- **Dashboard** — [app.agentx.so](https://app.agentx.so)
|
|
@@ -314,3 +340,4 @@ See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder,
|
|
|
314
340
|
- **Tracing docs** — [TRACING.md](TRACING.md)
|
|
315
341
|
- **Evaluations docs** — [EVALUATIONS.md](EVALUATIONS.md)
|
|
316
342
|
- **Monitor docs** — [docs.agentx.so/sdk/monitor](https://docs.agentx.so/sdk/monitor)
|
|
343
|
+
- **Self-host** — [AgentX-trace-eval](https://github.com/AgentX-ai/AgentX-trace-eval)
|
|
@@ -3,6 +3,7 @@ README.md
|
|
|
3
3
|
setup.py
|
|
4
4
|
agentx/__init__.py
|
|
5
5
|
agentx/agentx.py
|
|
6
|
+
agentx/cli.py
|
|
6
7
|
agentx/exceptions.py
|
|
7
8
|
agentx/py.typed
|
|
8
9
|
agentx/util.py
|
|
@@ -34,6 +35,7 @@ agentx/monitor/__init__.py
|
|
|
34
35
|
agentx/monitor/client.py
|
|
35
36
|
agentx/monitor/models.py
|
|
36
37
|
agentx/monitor/patterns.py
|
|
38
|
+
agentx/monitor/profile.py
|
|
37
39
|
agentx/monitor/signals.py
|
|
38
40
|
agentx/resources/__init__.py
|
|
39
41
|
agentx/resources/agent.py
|
|
@@ -46,6 +48,7 @@ agentx/tracing/tracer.py
|
|
|
46
48
|
agentx_python.egg-info/PKG-INFO
|
|
47
49
|
agentx_python.egg-info/SOURCES.txt
|
|
48
50
|
agentx_python.egg-info/dependency_links.txt
|
|
51
|
+
agentx_python.egg-info/entry_points.txt
|
|
49
52
|
agentx_python.egg-info/not-zip-safe
|
|
50
53
|
agentx_python.egg-info/requires.txt
|
|
51
54
|
agentx_python.egg-info/top_level.txt
|
|
@@ -26,6 +26,12 @@ setup(
|
|
|
26
26
|
# checking for anything that touches the SDK.
|
|
27
27
|
package_data={"agentx": ["py.typed"]},
|
|
28
28
|
zip_safe=False,
|
|
29
|
+
entry_points={
|
|
30
|
+
# Thin launcher for the self-hostable governance engine (AgentX-ai/AgentX-trace-eval),
|
|
31
|
+
# not this SDK itself, see agentx/cli.py's module docstring for why it lazily downloads
|
|
32
|
+
# rather than bundling that engine in this package.
|
|
33
|
+
"console_scripts": ["agentx-trace-eval=agentx.cli:main"],
|
|
34
|
+
},
|
|
29
35
|
install_requires=[
|
|
30
36
|
"urllib3>=1.26.11",
|
|
31
37
|
"certifi",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
VERSION = "0.6.9"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|