agentx-python 0.6.10__tar.gz → 0.6.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.10/agentx_python.egg-info → agentx_python-0.6.11}/PKG-INFO +34 -2
- {agentx_python-0.6.10 → agentx_python-0.6.11}/README.md +19 -1
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/client.py +25 -1
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/models.py +24 -0
- agentx_python-0.6.11/agentx/evaluations/prompts.py +49 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/runner.py +1 -0
- agentx_python-0.6.11/agentx/integrations/_traced_call.py +141 -0
- agentx_python-0.6.11/agentx/integrations/anthropic.py +266 -0
- agentx_python-0.6.11/agentx/integrations/autogen.py +199 -0
- agentx_python-0.6.11/agentx/integrations/crewai.py +257 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/google_adk.py +31 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/google_genai.py +127 -64
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/langchain.py +69 -7
- agentx_python-0.6.11/agentx/integrations/litellm.py +142 -0
- agentx_python-0.6.11/agentx/integrations/llamaindex.py +302 -0
- agentx_python-0.6.11/agentx/integrations/openai.py +158 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/openai_agents.py +11 -0
- agentx_python-0.6.11/agentx/version.py +1 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11/agentx_python.egg-info}/PKG-INFO +34 -2
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/SOURCES.txt +8 -1
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/requires.txt +18 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/setup.py +9 -0
- agentx_python-0.6.11/tests/test_integrations.py +1011 -0
- agentx_python-0.6.10/agentx/integrations/anthropic.py +0 -250
- agentx_python-0.6.10/agentx/integrations/crewai.py +0 -141
- agentx_python-0.6.10/agentx/version.py +0 -1
- {agentx_python-0.6.10 → agentx_python-0.6.11}/LICENSE +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/agentx.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/cli.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/exceptions.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/integrations/_perf.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/client.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/models.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/profile.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/py.typed +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/tracing/tracer.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx/util.py +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/entry_points.txt +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/setup.cfg +0 -0
- {agentx_python-0.6.10 → agentx_python-0.6.11}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.11
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -21,19 +21,33 @@ Provides-Extra: crewai
|
|
|
21
21
|
Requires-Dist: crewai>=0.80.0; extra == "crewai"
|
|
22
22
|
Provides-Extra: openai-agents
|
|
23
23
|
Requires-Dist: openai-agents>=0.0.3; extra == "openai-agents"
|
|
24
|
+
Provides-Extra: openai
|
|
25
|
+
Requires-Dist: openai>=1.0.0; extra == "openai"
|
|
24
26
|
Provides-Extra: anthropic
|
|
25
27
|
Requires-Dist: anthropic>=0.25.0; extra == "anthropic"
|
|
26
28
|
Provides-Extra: google-adk
|
|
27
29
|
Requires-Dist: google-adk>=1.0.0; extra == "google-adk"
|
|
28
30
|
Provides-Extra: google-genai
|
|
29
31
|
Requires-Dist: google-genai>=1.0.0; extra == "google-genai"
|
|
32
|
+
Provides-Extra: litellm
|
|
33
|
+
Requires-Dist: litellm>=1.0.0; extra == "litellm"
|
|
34
|
+
Provides-Extra: llamaindex
|
|
35
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == "llamaindex"
|
|
36
|
+
Provides-Extra: autogen
|
|
37
|
+
Requires-Dist: autogen-agentchat>=0.4.0; extra == "autogen"
|
|
38
|
+
Requires-Dist: autogen-core>=0.4.0; extra == "autogen"
|
|
30
39
|
Provides-Extra: all
|
|
31
40
|
Requires-Dist: langchain-core>=0.1.0; extra == "all"
|
|
32
41
|
Requires-Dist: crewai>=0.80.0; extra == "all"
|
|
33
42
|
Requires-Dist: openai-agents>=0.0.3; extra == "all"
|
|
43
|
+
Requires-Dist: openai>=1.0.0; extra == "all"
|
|
34
44
|
Requires-Dist: anthropic>=0.25.0; extra == "all"
|
|
35
45
|
Requires-Dist: google-adk>=1.0.0; extra == "all"
|
|
36
46
|
Requires-Dist: google-genai>=1.0.0; extra == "all"
|
|
47
|
+
Requires-Dist: litellm>=1.0.0; extra == "all"
|
|
48
|
+
Requires-Dist: llama-index-core>=0.10.0; extra == "all"
|
|
49
|
+
Requires-Dist: autogen-agentchat>=0.4.0; extra == "all"
|
|
50
|
+
Requires-Dist: autogen-core>=0.4.0; extra == "all"
|
|
37
51
|
Dynamic: author
|
|
38
52
|
Dynamic: author-email
|
|
39
53
|
Dynamic: classifier
|
|
@@ -235,9 +249,13 @@ Each integration auto-captures LLM calls, tool calls, and token usage — instal
|
|
|
235
249
|
| LangChain | `pip install "agentx-python[langchain]"` | `AgentXCallbackHandler` |
|
|
236
250
|
| CrewAI | `pip install "agentx-python[crewai]"` | `AgentXCrewObserver` |
|
|
237
251
|
| OpenAI Agents SDK | `pip install "agentx-python[openai-agents]"` | `AgentXTracingProcessor` |
|
|
252
|
+
| OpenAI (raw client) | `pip install "agentx-python[openai]"` | `patch_openai_client` |
|
|
238
253
|
| Anthropic | `pip install "agentx-python[anthropic]"` | `patch_anthropic_client` |
|
|
239
254
|
| Google ADK | `pip install "agentx-python[google-adk]"` | `AgentXADKPlugin` |
|
|
240
255
|
| Google GenAI (Gemini) | `pip install "agentx-python[google-genai]"` | `patch_genai_client` |
|
|
256
|
+
| LiteLLM | `pip install "agentx-python[litellm]"` | `AgentXLiteLLMLogger` |
|
|
257
|
+
| LlamaIndex | `pip install "agentx-python[llamaindex]"` | `AgentXLlamaIndexHandler`|
|
|
258
|
+
| AutoGen | `pip install "agentx-python[autogen]"` | `AgentXAutoGenObserver` |
|
|
241
259
|
|
|
242
260
|
Or plain Python — wrap any function with `@tracer.trace(...)` and it just works, no framework required.
|
|
243
261
|
|
|
@@ -309,7 +327,21 @@ print(report.recommendations) # list of prioritized, actionable fixes
|
|
|
309
327
|
|
|
310
328
|
Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
|
|
311
329
|
|
|
312
|
-
|
|
330
|
+
Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
prompt = client.evaluations.prompts.get("support-agent-system-prompt")
|
|
334
|
+
# use prompt.text as your own agent's system prompt
|
|
335
|
+
|
|
336
|
+
client.evaluations.run(
|
|
337
|
+
dataset_id="evds_…",
|
|
338
|
+
subject={"kind": "custom_agent", "metadata": {"promptName": prompt.name}},
|
|
339
|
+
).execute(my_agent_fn)
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/self-host#prompt-registry) for the "Suggest improvement" dashboard flow (self-host only — no hosted-SaaS equivalent yet).
|
|
343
|
+
|
|
344
|
+
See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
|
|
313
345
|
|
|
314
346
|
---
|
|
315
347
|
|
|
@@ -187,9 +187,13 @@ Each integration auto-captures LLM calls, tool calls, and token usage — instal
|
|
|
187
187
|
| LangChain | `pip install "agentx-python[langchain]"` | `AgentXCallbackHandler` |
|
|
188
188
|
| CrewAI | `pip install "agentx-python[crewai]"` | `AgentXCrewObserver` |
|
|
189
189
|
| OpenAI Agents SDK | `pip install "agentx-python[openai-agents]"` | `AgentXTracingProcessor` |
|
|
190
|
+
| OpenAI (raw client) | `pip install "agentx-python[openai]"` | `patch_openai_client` |
|
|
190
191
|
| Anthropic | `pip install "agentx-python[anthropic]"` | `patch_anthropic_client` |
|
|
191
192
|
| Google ADK | `pip install "agentx-python[google-adk]"` | `AgentXADKPlugin` |
|
|
192
193
|
| Google GenAI (Gemini) | `pip install "agentx-python[google-genai]"` | `patch_genai_client` |
|
|
194
|
+
| LiteLLM | `pip install "agentx-python[litellm]"` | `AgentXLiteLLMLogger` |
|
|
195
|
+
| LlamaIndex | `pip install "agentx-python[llamaindex]"` | `AgentXLlamaIndexHandler`|
|
|
196
|
+
| AutoGen | `pip install "agentx-python[autogen]"` | `AgentXAutoGenObserver` |
|
|
193
197
|
|
|
194
198
|
Or plain Python — wrap any function with `@tracer.trace(...)` and it just works, no framework required.
|
|
195
199
|
|
|
@@ -261,7 +265,21 @@ print(report.recommendations) # list of prioritized, actionable fixes
|
|
|
261
265
|
|
|
262
266
|
Ask a case's question several extra ways each run, LLM-paraphrased server-side, to catch agents that break on phrasing rather than substance, and override the judge's prompt/model per config, see [Smoke testing](EVALUATIONS.md#smoke-testing-phrasing-robustness) and [Configuring the judge](EVALUATIONS.md#configuring-the-judge) in the full guide.
|
|
263
267
|
|
|
264
|
-
|
|
268
|
+
Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
|
|
269
|
+
|
|
270
|
+
```python
|
|
271
|
+
prompt = client.evaluations.prompts.get("support-agent-system-prompt")
|
|
272
|
+
# use prompt.text as your own agent's system prompt
|
|
273
|
+
|
|
274
|
+
client.evaluations.run(
|
|
275
|
+
dataset_id="evds_…",
|
|
276
|
+
subject={"kind": "custom_agent", "metadata": {"promptName": prompt.name}},
|
|
277
|
+
).execute(my_agent_fn)
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
See [Prompt registry](EVALUATIONS.md#prompt-registry) in the full guide, or [self-host's docs](https://docs.agentx.so/self-host#prompt-registry) for the "Suggest improvement" dashboard flow (self-host only — no hosted-SaaS equivalent yet).
|
|
281
|
+
|
|
282
|
+
See **[EVALUATIONS.md](EVALUATIONS.md)** for the full guide — dataset builder, framework adapters, similarity metrics, smoke testing, judge configuration, prompt registry, and the complete API reference.
|
|
265
283
|
|
|
266
284
|
---
|
|
267
285
|
|
|
@@ -17,6 +17,7 @@ from agentx.evaluations.models import (
|
|
|
17
17
|
EvaluationSettings,
|
|
18
18
|
EvaluationSubject,
|
|
19
19
|
ModelInfo,
|
|
20
|
+
Prompt,
|
|
20
21
|
Report,
|
|
21
22
|
)
|
|
22
23
|
|
|
@@ -77,12 +78,14 @@ class EvaluationsClient:
|
|
|
77
78
|
"accept": "*/*",
|
|
78
79
|
}
|
|
79
80
|
)
|
|
80
|
-
# Expose dataset / evaluation-settings builder factories
|
|
81
|
+
# Expose dataset / evaluation-settings / prompt builder factories
|
|
81
82
|
from agentx.evaluations.datasets import DatasetClient
|
|
82
83
|
from agentx.evaluations.evaluation_settings import EvaluationSettingsClient
|
|
84
|
+
from agentx.evaluations.prompts import PromptClient
|
|
83
85
|
|
|
84
86
|
self.datasets = DatasetClient(self)
|
|
85
87
|
self.settings = EvaluationSettingsClient(self)
|
|
88
|
+
self.prompts = PromptClient(self)
|
|
86
89
|
|
|
87
90
|
# ------------------------------------------------------------------
|
|
88
91
|
# Low-level HTTP
|
|
@@ -195,6 +198,27 @@ class EvaluationsClient:
|
|
|
195
198
|
)
|
|
196
199
|
return EvaluationSettings(**data)
|
|
197
200
|
|
|
201
|
+
# ------------------------------------------------------------------
|
|
202
|
+
# Prompt registry endpoints — see agentx.evaluations.prompts.PromptClient for the concept
|
|
203
|
+
# (the external-agent analog to native autotune). Deliberately read-mostly: no publish here,
|
|
204
|
+
# a new version only ever comes from the dashboard's human-approved propose/publish flow.
|
|
205
|
+
# ------------------------------------------------------------------
|
|
206
|
+
|
|
207
|
+
def create_prompt(self, payload: dict) -> Prompt:
|
|
208
|
+
data = self._request("POST", "/prompts", json=self._with_workspace(payload))
|
|
209
|
+
return Prompt(**data)
|
|
210
|
+
|
|
211
|
+
def list_prompts(self) -> List[Prompt]:
|
|
212
|
+
data = self._request("GET", "/prompts", params=self._workspace_params())
|
|
213
|
+
return [Prompt(**p) for p in (data if isinstance(data, list) else data.get("prompts", []))]
|
|
214
|
+
|
|
215
|
+
def get_prompt(self, name: str, version: Optional[int] = None) -> Prompt:
|
|
216
|
+
params = self._workspace_params() or {}
|
|
217
|
+
if version is not None:
|
|
218
|
+
params = {**params, "version": version}
|
|
219
|
+
data = self._request("GET", f"/prompts/{name}", params=params or None)
|
|
220
|
+
return Prompt(**data)
|
|
221
|
+
|
|
198
222
|
# ------------------------------------------------------------------
|
|
199
223
|
# Run endpoints
|
|
200
224
|
# ------------------------------------------------------------------
|
|
@@ -128,6 +128,30 @@ class EvaluationSettings(BaseModel):
|
|
|
128
128
|
extra = "ignore"
|
|
129
129
|
|
|
130
130
|
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
# Prompt registry — the external-agent analog to AgentX's native autotune. AgentX doesn't own
|
|
133
|
+
# your agent's code, so instead of branching/applying a config it becomes the prompt's source of
|
|
134
|
+
# truth (same shape as LangSmith's Prompt Hub / Langfuse's Prompt Management): pull a version at
|
|
135
|
+
# runtime with ``client.evaluations.prompts.get(name)``, use ``prompt.text`` as your agent's
|
|
136
|
+
# system prompt, and tag your eval runs so the existing version-comparison view can tell you
|
|
137
|
+
# which published version scored higher — see ``client.evaluations.prompts`` docs.
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class Prompt(BaseModel):
|
|
142
|
+
id: str = Field(alias="_id")
|
|
143
|
+
name: str
|
|
144
|
+
description: Optional[str] = None
|
|
145
|
+
version: int
|
|
146
|
+
text: str
|
|
147
|
+
created_at: Optional[str] = Field(default=None, alias="createdAt")
|
|
148
|
+
updated_at: Optional[str] = Field(default=None, alias="updatedAt")
|
|
149
|
+
|
|
150
|
+
class Config:
|
|
151
|
+
populate_by_name = True
|
|
152
|
+
extra = "ignore"
|
|
153
|
+
|
|
154
|
+
|
|
131
155
|
# ---------------------------------------------------------------------------
|
|
132
156
|
# Evaluation subject
|
|
133
157
|
# ---------------------------------------------------------------------------
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, List, Optional
|
|
4
|
+
|
|
5
|
+
from agentx.evaluations.models import Prompt
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from agentx.evaluations.client import EvaluationsClient
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PromptClient:
|
|
12
|
+
"""Thin wrapper surfaced as ``client.evaluations.prompts`` — the external-agent analog to
|
|
13
|
+
AgentX's native autotune. AgentX doesn't own your agent's code, so instead of branching and
|
|
14
|
+
applying a config, it becomes the prompt's source of truth (the same shape as LangSmith's
|
|
15
|
+
Prompt Hub / Langfuse's Prompt Management): create/pull versions here, use ``prompt.text`` as
|
|
16
|
+
your agent's own system prompt, and tag your eval runs with the pulled version so the
|
|
17
|
+
existing version-comparison view (``client.evaluations`` run comparisons on a dataset) can
|
|
18
|
+
tell you which published version actually scored higher.
|
|
19
|
+
|
|
20
|
+
Deliberately read-mostly from here: there is no ``publish`` on this client. A prompt only
|
|
21
|
+
gets a new version through the dashboard's human-approved propose/publish flow, so a
|
|
22
|
+
rewritten prompt never reaches your running agent without someone explicitly approving it.
|
|
23
|
+
|
|
24
|
+
Example::
|
|
25
|
+
|
|
26
|
+
prompt = client.evaluations.prompts.get("support-agent-system-prompt")
|
|
27
|
+
# ... call your own LLM using prompt.text as the system prompt ...
|
|
28
|
+
client.evaluations.init_run(
|
|
29
|
+
dataset_id=dataset_id,
|
|
30
|
+
subject=EvaluationSubject(
|
|
31
|
+
metadata={
|
|
32
|
+
"promptName": prompt.name,
|
|
33
|
+
"version": f"{prompt.name}@v{prompt.version}",
|
|
34
|
+
}
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, client: "EvaluationsClient"):
|
|
40
|
+
self._client = client
|
|
41
|
+
|
|
42
|
+
def create(self, name: str, text: str, description: Optional[str] = None) -> Prompt:
|
|
43
|
+
return self._client.create_prompt({"name": name, "text": text, "description": description})
|
|
44
|
+
|
|
45
|
+
def get(self, name: str, version: Optional[int] = None) -> Prompt:
|
|
46
|
+
return self._client.get_prompt(name, version=version)
|
|
47
|
+
|
|
48
|
+
def list(self) -> List[Prompt]:
|
|
49
|
+
return self._client.list_prompts()
|
|
@@ -335,6 +335,7 @@ class EvaluationsRunner:
|
|
|
335
335
|
self._client = client
|
|
336
336
|
self.datasets = client.datasets
|
|
337
337
|
self.settings = client.settings
|
|
338
|
+
self.prompts = client.prompts
|
|
338
339
|
|
|
339
340
|
def list_models(self, provider: Optional[str] = None) -> List[ModelInfo]:
|
|
340
341
|
"""List the LLM models AgentX supports — the same set selectable for
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared helper for tracing a raw SDK client call that may be sync or async.
|
|
3
|
+
|
|
4
|
+
Anthropic's and Google GenAI's raw clients expose the same method names for
|
|
5
|
+
both their sync and async client variants (``client.messages.create``,
|
|
6
|
+
``client.models.generate_content``) — the only way to tell them apart is to
|
|
7
|
+
call the method and check whether the result is awaitable.
|
|
8
|
+
``inspect.iscoroutinefunction`` is unreliable for this: it returns ``False``
|
|
9
|
+
even for ``AsyncAnthropic().messages.create``, since these SDKs don't
|
|
10
|
+
implement the async variant as a plain top-level ``async def``.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
import inspect
|
|
16
|
+
from typing import Any, Callable, Dict, Optional
|
|
17
|
+
|
|
18
|
+
from agentx.tracing.tracer import Tracer
|
|
19
|
+
from agentx.integrations._perf import build_performance_summary
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def call_and_trace(
|
|
23
|
+
original: Callable[..., Any],
|
|
24
|
+
args: tuple,
|
|
25
|
+
kwargs: dict,
|
|
26
|
+
on_finish: Callable[[Optional[Any], Optional[str]], None],
|
|
27
|
+
) -> Any:
|
|
28
|
+
"""
|
|
29
|
+
Call ``original(*args, **kwargs)``.
|
|
30
|
+
|
|
31
|
+
If the result is awaitable (async client), return a coroutine that awaits
|
|
32
|
+
it and calls ``on_finish(response, error)`` only after the real await
|
|
33
|
+
completes, so timing/output/tokens reflect the actual call rather than
|
|
34
|
+
the moment the coroutine object was constructed. If the result is a
|
|
35
|
+
normal value (sync client), call ``on_finish`` immediately.
|
|
36
|
+
|
|
37
|
+
Either way, the original call's own return value / exception behavior is
|
|
38
|
+
unchanged for the caller — this only affects when/how the trace is built.
|
|
39
|
+
"""
|
|
40
|
+
try:
|
|
41
|
+
result = original(*args, **kwargs)
|
|
42
|
+
except Exception as exc:
|
|
43
|
+
on_finish(None, str(exc))
|
|
44
|
+
raise
|
|
45
|
+
|
|
46
|
+
if asyncio.iscoroutine(result) or inspect.isawaitable(result):
|
|
47
|
+
return _await_and_finish(result, on_finish)
|
|
48
|
+
|
|
49
|
+
on_finish(result, None)
|
|
50
|
+
return result
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def _await_and_finish(
|
|
54
|
+
awaitable: Any,
|
|
55
|
+
on_finish: Callable[[Optional[Any], Optional[str]], None],
|
|
56
|
+
) -> Any:
|
|
57
|
+
try:
|
|
58
|
+
response = await awaitable
|
|
59
|
+
except Exception as exc:
|
|
60
|
+
on_finish(None, str(exc))
|
|
61
|
+
raise
|
|
62
|
+
on_finish(response, None)
|
|
63
|
+
return response
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def finish_llm_call(
|
|
67
|
+
tracer: Tracer,
|
|
68
|
+
*,
|
|
69
|
+
name: str,
|
|
70
|
+
framework: str,
|
|
71
|
+
metadata: Optional[Dict[str, Any]],
|
|
72
|
+
session_id: Optional[str],
|
|
73
|
+
start_t: float,
|
|
74
|
+
end_t: float,
|
|
75
|
+
input_repr: Any,
|
|
76
|
+
output: Optional[str],
|
|
77
|
+
model: Optional[str],
|
|
78
|
+
input_tokens: Optional[int],
|
|
79
|
+
output_tokens: Optional[int],
|
|
80
|
+
error: Optional[str],
|
|
81
|
+
) -> None:
|
|
82
|
+
"""
|
|
83
|
+
Close out one raw-client LLM call — shared by the ``on_finish``/exit
|
|
84
|
+
callbacks of every integration that patches a raw provider client
|
|
85
|
+
(``anthropic.py``, ``google_genai.py``, ``openai.py``) rather than a
|
|
86
|
+
framework-level callback/plugin system.
|
|
87
|
+
|
|
88
|
+
If the call happened inside a ``with tracer.trace(...)`` block, attach it
|
|
89
|
+
as one LLM-call step on that span instead of sending an independent
|
|
90
|
+
trace — the same "part of a multi-call agentic loop" behavior
|
|
91
|
+
``anthropic.py`` already had; folded in here so every raw-client
|
|
92
|
+
integration gets it instead of each having to remember to check
|
|
93
|
+
``tracer.current_span`` itself.
|
|
94
|
+
"""
|
|
95
|
+
latency_ms = int((end_t - start_t) * 1000)
|
|
96
|
+
|
|
97
|
+
active_span = tracer.current_span
|
|
98
|
+
if active_span is not None:
|
|
99
|
+
if error is not None:
|
|
100
|
+
active_span.set_error(error)
|
|
101
|
+
active_span._record_llm_call(
|
|
102
|
+
duration_ms=latency_ms,
|
|
103
|
+
start_time=start_t,
|
|
104
|
+
end_time=end_t,
|
|
105
|
+
input=input_repr,
|
|
106
|
+
output=output,
|
|
107
|
+
model=model,
|
|
108
|
+
input_tokens=input_tokens,
|
|
109
|
+
output_tokens=output_tokens,
|
|
110
|
+
)
|
|
111
|
+
return
|
|
112
|
+
|
|
113
|
+
perf = build_performance_summary(
|
|
114
|
+
total_duration_ms=latency_ms,
|
|
115
|
+
execution_steps=[{
|
|
116
|
+
"name": "LLM Call 1",
|
|
117
|
+
"duration_ms": latency_ms,
|
|
118
|
+
"start_time": start_t,
|
|
119
|
+
"end_time": end_t,
|
|
120
|
+
"model": model,
|
|
121
|
+
"input": input_repr,
|
|
122
|
+
"output": output,
|
|
123
|
+
"inputTokenSize": input_tokens,
|
|
124
|
+
"outputTokenSize": output_tokens,
|
|
125
|
+
}],
|
|
126
|
+
has_errors=error is not None,
|
|
127
|
+
)
|
|
128
|
+
tracer._send(
|
|
129
|
+
name=name,
|
|
130
|
+
input=input_repr,
|
|
131
|
+
output=output,
|
|
132
|
+
latency_ms=latency_ms,
|
|
133
|
+
error=error,
|
|
134
|
+
framework=framework,
|
|
135
|
+
model=model,
|
|
136
|
+
metadata=metadata,
|
|
137
|
+
session_id=session_id,
|
|
138
|
+
performance_summary=perf,
|
|
139
|
+
input_tokens=input_tokens,
|
|
140
|
+
output_tokens=output_tokens,
|
|
141
|
+
)
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Anthropic SDK integration for AgentX production tracing.
|
|
3
|
+
|
|
4
|
+
Usage::
|
|
5
|
+
|
|
6
|
+
from agentx.integrations.anthropic import patch_anthropic_client
|
|
7
|
+
import anthropic
|
|
8
|
+
|
|
9
|
+
client = anthropic.Anthropic()
|
|
10
|
+
patch_anthropic_client(client, agentx.tracer, name="claude-agent")
|
|
11
|
+
|
|
12
|
+
# All subsequent client.messages.create() calls are now traced automatically.
|
|
13
|
+
|
|
14
|
+
Works with both ``anthropic.Anthropic`` and ``anthropic.AsyncAnthropic`` clients.
|
|
15
|
+
|
|
16
|
+
Requires: ``pip install agentx[anthropic]``
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import inspect
|
|
21
|
+
import time
|
|
22
|
+
from typing import Any, Dict, Optional, Tuple
|
|
23
|
+
|
|
24
|
+
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
25
|
+
from agentx.integrations._traced_call import call_and_trace, finish_llm_call
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _extract_output_text(response: Any) -> Optional[str]:
|
|
29
|
+
"""
|
|
30
|
+
Extract the assistant's text reply from a Messages API response, falling
|
|
31
|
+
back to a description of any tool_use blocks when the response is a pure
|
|
32
|
+
tool call with no accompanying text.
|
|
33
|
+
"""
|
|
34
|
+
content = getattr(response, "content", None) if response is not None else None
|
|
35
|
+
if not content:
|
|
36
|
+
return None
|
|
37
|
+
texts = []
|
|
38
|
+
tool_calls = []
|
|
39
|
+
for block in content:
|
|
40
|
+
block_type = getattr(block, "type", None)
|
|
41
|
+
if block_type == "text":
|
|
42
|
+
text = getattr(block, "text", None)
|
|
43
|
+
if text:
|
|
44
|
+
texts.append(text)
|
|
45
|
+
elif block_type == "tool_use":
|
|
46
|
+
name = getattr(block, "name", "unknown")
|
|
47
|
+
tool_input = getattr(block, "input", None)
|
|
48
|
+
tool_calls.append(f"{name}({tool_input})")
|
|
49
|
+
if texts:
|
|
50
|
+
return "\n".join(texts)
|
|
51
|
+
if tool_calls:
|
|
52
|
+
return "[tool call] " + ", ".join(tool_calls)
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _prepend_system(messages: Any, system: Any) -> Any:
|
|
57
|
+
"""
|
|
58
|
+
Fold the ``system`` kwarg (a separate top-level parameter in the Anthropic
|
|
59
|
+
SDK, not part of ``messages``) into the traced input as a leading
|
|
60
|
+
system-role entry — the same shape trace consumers already expect from
|
|
61
|
+
other frameworks' captured input.
|
|
62
|
+
"""
|
|
63
|
+
if not system:
|
|
64
|
+
return messages
|
|
65
|
+
system_entry = {"role": "system", "content": system}
|
|
66
|
+
if isinstance(messages, list):
|
|
67
|
+
return [system_entry] + list(messages)
|
|
68
|
+
if messages is None:
|
|
69
|
+
return [system_entry]
|
|
70
|
+
return [system_entry, messages]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _extract_usage_tokens(usage: Any) -> Tuple[Optional[int], Optional[int]]:
|
|
74
|
+
"""
|
|
75
|
+
Pull input/output token counts off a ``response.usage`` object, folding
|
|
76
|
+
prompt-caching tokens (``cache_creation_input_tokens`` /
|
|
77
|
+
``cache_read_input_tokens``) into the input total — they're still real
|
|
78
|
+
input tokens for cost/context-window purposes, and the backend has no
|
|
79
|
+
separate column for them today.
|
|
80
|
+
"""
|
|
81
|
+
if usage is None:
|
|
82
|
+
return None, None
|
|
83
|
+
input_tokens = getattr(usage, "input_tokens", None)
|
|
84
|
+
output_tokens = getattr(usage, "output_tokens", None)
|
|
85
|
+
cache_creation = getattr(usage, "cache_creation_input_tokens", None)
|
|
86
|
+
cache_read = getattr(usage, "cache_read_input_tokens", None)
|
|
87
|
+
if cache_creation or cache_read:
|
|
88
|
+
input_tokens = (input_tokens or 0) + (cache_creation or 0) + (cache_read or 0)
|
|
89
|
+
return input_tokens, output_tokens
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def patch_anthropic_client(
|
|
93
|
+
client: Any,
|
|
94
|
+
tracer: Tracer,
|
|
95
|
+
name: str = "anthropic-agent",
|
|
96
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
97
|
+
session_id: Optional[str] = None,
|
|
98
|
+
) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Monkey-patch ``client.messages.create`` and ``client.messages.stream``
|
|
101
|
+
(if present) to automatically send a trace for every call.
|
|
102
|
+
|
|
103
|
+
The original method is still called and its return value is passed through
|
|
104
|
+
unchanged so nothing in the caller needs to change. Works with both sync
|
|
105
|
+
(``Anthropic``) and async (``AsyncAnthropic``) clients.
|
|
106
|
+
"""
|
|
107
|
+
messages = getattr(client, "messages", None)
|
|
108
|
+
if messages is None:
|
|
109
|
+
raise ValueError("Provided client does not have a .messages attribute")
|
|
110
|
+
|
|
111
|
+
_patch_create(messages, tracer, name, metadata, session_id)
|
|
112
|
+
|
|
113
|
+
# stream is optional (not present in all versions)
|
|
114
|
+
if hasattr(messages, "stream"):
|
|
115
|
+
_patch_stream(messages, tracer, name, metadata, session_id)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _patch_create(
|
|
119
|
+
messages_resource: Any,
|
|
120
|
+
tracer: Tracer,
|
|
121
|
+
name: str,
|
|
122
|
+
metadata: Optional[Dict[str, Any]],
|
|
123
|
+
session_id: Optional[str],
|
|
124
|
+
) -> None:
|
|
125
|
+
original = messages_resource.create
|
|
126
|
+
if getattr(original, "_agentx_patched", False):
|
|
127
|
+
return # already patched
|
|
128
|
+
|
|
129
|
+
def patched_create(*args, **kwargs):
|
|
130
|
+
start_t = time.time()
|
|
131
|
+
input_messages = _prepend_system(
|
|
132
|
+
kwargs.get("messages") or (args[0] if args else None),
|
|
133
|
+
kwargs.get("system"),
|
|
134
|
+
)
|
|
135
|
+
model = kwargs.get("model")
|
|
136
|
+
|
|
137
|
+
input_repr = _safe_serialize(input_messages)
|
|
138
|
+
|
|
139
|
+
def on_finish(response: Optional[Any], error: Optional[str]) -> None:
|
|
140
|
+
end_t = time.time()
|
|
141
|
+
output = None
|
|
142
|
+
input_tokens = None
|
|
143
|
+
output_tokens = None
|
|
144
|
+
if response is not None:
|
|
145
|
+
output = _extract_output_text(response)
|
|
146
|
+
try:
|
|
147
|
+
input_tokens, output_tokens = _extract_usage_tokens(getattr(response, "usage", None))
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
|
|
151
|
+
finish_llm_call(
|
|
152
|
+
tracer,
|
|
153
|
+
name=name,
|
|
154
|
+
framework="anthropic",
|
|
155
|
+
metadata=metadata,
|
|
156
|
+
session_id=session_id,
|
|
157
|
+
start_t=start_t,
|
|
158
|
+
end_t=end_t,
|
|
159
|
+
input_repr=input_repr,
|
|
160
|
+
output=output,
|
|
161
|
+
model=model,
|
|
162
|
+
input_tokens=input_tokens,
|
|
163
|
+
output_tokens=output_tokens,
|
|
164
|
+
error=error,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
return call_and_trace(original, args, kwargs, on_finish)
|
|
168
|
+
|
|
169
|
+
patched_create._agentx_patched = True
|
|
170
|
+
messages_resource.create = patched_create
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _patch_stream(
|
|
174
|
+
messages_resource: Any,
|
|
175
|
+
tracer: Tracer,
|
|
176
|
+
name: str,
|
|
177
|
+
metadata: Optional[Dict[str, Any]],
|
|
178
|
+
session_id: Optional[str],
|
|
179
|
+
) -> None:
|
|
180
|
+
original_stream = messages_resource.stream
|
|
181
|
+
if getattr(original_stream, "_agentx_patched", False):
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
def patched_stream(*args, **kwargs):
|
|
185
|
+
# `.stream()` itself returns a context-manager object synchronously
|
|
186
|
+
# for both `Anthropic` and `AsyncAnthropic` — the async/sync split
|
|
187
|
+
# only shows up in whether `with`/`async with` and
|
|
188
|
+
# `get_final_message()` are used, handled inside `_TracedStream`.
|
|
189
|
+
start_t = time.time()
|
|
190
|
+
ctx = original_stream(*args, **kwargs)
|
|
191
|
+
input_repr = _safe_serialize(_prepend_system(kwargs.get("messages"), kwargs.get("system")))
|
|
192
|
+
model = kwargs.get("model")
|
|
193
|
+
|
|
194
|
+
def build_and_send(end_t: float, error: Optional[str], final_message: Optional[Any]) -> None:
|
|
195
|
+
output = None
|
|
196
|
+
input_tokens = None
|
|
197
|
+
output_tokens = None
|
|
198
|
+
if final_message is not None:
|
|
199
|
+
output = _extract_output_text(final_message)
|
|
200
|
+
try:
|
|
201
|
+
input_tokens, output_tokens = _extract_usage_tokens(getattr(final_message, "usage", None))
|
|
202
|
+
except Exception:
|
|
203
|
+
pass
|
|
204
|
+
finish_llm_call(
|
|
205
|
+
tracer,
|
|
206
|
+
name=name,
|
|
207
|
+
framework="anthropic",
|
|
208
|
+
metadata=metadata,
|
|
209
|
+
session_id=session_id,
|
|
210
|
+
start_t=start_t,
|
|
211
|
+
end_t=end_t,
|
|
212
|
+
input_repr=input_repr,
|
|
213
|
+
output=output,
|
|
214
|
+
model=model,
|
|
215
|
+
input_tokens=input_tokens,
|
|
216
|
+
output_tokens=output_tokens,
|
|
217
|
+
error=error,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
class _TracedStream:
|
|
221
|
+
"""Thin wrapper that records timing when the stream context exits."""
|
|
222
|
+
|
|
223
|
+
def __enter__(self_inner):
|
|
224
|
+
return ctx.__enter__()
|
|
225
|
+
|
|
226
|
+
def __exit__(self_inner, exc_type, exc_val, tb):
|
|
227
|
+
result = ctx.__exit__(exc_type, exc_val, tb)
|
|
228
|
+
end_t = time.time()
|
|
229
|
+
error = str(exc_val) if exc_val else None
|
|
230
|
+
final_message = None
|
|
231
|
+
try:
|
|
232
|
+
final_message = ctx.get_final_message()
|
|
233
|
+
except Exception:
|
|
234
|
+
pass
|
|
235
|
+
build_and_send(end_t, error, final_message)
|
|
236
|
+
return result
|
|
237
|
+
|
|
238
|
+
async def __aenter__(self_inner):
|
|
239
|
+
return await ctx.__aenter__()
|
|
240
|
+
|
|
241
|
+
async def __aexit__(self_inner, exc_type, exc_val, tb):
|
|
242
|
+
result = await ctx.__aexit__(exc_type, exc_val, tb)
|
|
243
|
+
end_t = time.time()
|
|
244
|
+
error = str(exc_val) if exc_val else None
|
|
245
|
+
final_message = None
|
|
246
|
+
try:
|
|
247
|
+
raw = ctx.get_final_message()
|
|
248
|
+
final_message = await raw if inspect.isawaitable(raw) else raw
|
|
249
|
+
except Exception:
|
|
250
|
+
pass
|
|
251
|
+
build_and_send(end_t, error, final_message)
|
|
252
|
+
return result
|
|
253
|
+
|
|
254
|
+
def __iter__(self_inner):
|
|
255
|
+
return iter(ctx)
|
|
256
|
+
|
|
257
|
+
def __aiter__(self_inner):
|
|
258
|
+
return aiter(ctx)
|
|
259
|
+
|
|
260
|
+
def __getattr__(self_inner, item):
|
|
261
|
+
return getattr(ctx, item)
|
|
262
|
+
|
|
263
|
+
return _TracedStream()
|
|
264
|
+
|
|
265
|
+
patched_stream._agentx_patched = True
|
|
266
|
+
messages_resource.stream = patched_stream
|