agentx-python 0.6.11__tar.gz → 0.6.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentx_python-0.6.11/agentx_python.egg-info → agentx_python-0.6.13}/PKG-INFO +15 -2
- {agentx_python-0.6.11 → agentx_python-0.6.13}/README.md +14 -1
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/client.py +2 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/models.py +1 -1
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/prompts.py +6 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/_traced_call.py +20 -37
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/autogen.py +16 -20
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/crewai.py +19 -25
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/google_adk.py +52 -85
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/langchain.py +32 -61
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/llamaindex.py +24 -23
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/openai_agents.py +58 -73
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/client.py +49 -1
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/models.py +59 -0
- agentx_python-0.6.13/agentx/monitor/online_evaluators.py +120 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/tracing/tracer.py +233 -179
- agentx_python-0.6.13/agentx/version.py +1 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13/agentx_python.egg-info}/PKG-INFO +15 -2
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/SOURCES.txt +3 -2
- {agentx_python-0.6.11 → agentx_python-0.6.13}/tests/test_integrations.py +21 -16
- agentx_python-0.6.13/tests/test_span_tree.py +625 -0
- agentx_python-0.6.11/agentx/integrations/_perf.py +0 -226
- agentx_python-0.6.11/agentx/version.py +0 -1
- {agentx_python-0.6.11 → agentx_python-0.6.13}/LICENSE +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/agentx.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/cli.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/_term.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/adapters/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/adapters/http_endpoint.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/adapters/precomputed.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/adapters/raw.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/datasets.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/evaluation_settings.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/redaction.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/reporting.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/results.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/runner.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/evaluations/tracing.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/exceptions.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/anthropic.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/google_genai.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/litellm.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/integrations/openai.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/patterns.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/profile.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/monitor/signals.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/py.typed +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/resources/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/resources/agent.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/resources/conversation.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/resources/workforce.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/tracing/__init__.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/tracing/ci_types.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/tracing/ingest_client.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx/util.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/dependency_links.txt +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/entry_points.txt +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/not-zip-safe +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/requires.txt +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/agentx_python.egg-info/top_level.txt +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/setup.cfg +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/setup.py +0 -0
- {agentx_python-0.6.11 → agentx_python-0.6.13}/tests/test_integration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentx-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.13
|
|
4
4
|
Summary: Official Python SDK for AgentX (https://www.agentx.so/)
|
|
5
5
|
Home-page: https://github.com/AgentX-ai/AgentX-python
|
|
6
6
|
Author: Robin Wang and AgentX Team
|
|
@@ -298,6 +298,19 @@ Per-agent coverage/threshold settings (sample rate, retention, and threshold ove
|
|
|
298
298
|
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
299
299
|
```
|
|
300
300
|
|
|
301
|
+
Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
|
|
302
|
+
|
|
303
|
+
```python
|
|
304
|
+
evaluator = client.monitor.online_evaluators.builder(
|
|
305
|
+
name="Helpfulness",
|
|
306
|
+
evaluation_settings_id=settings.id,
|
|
307
|
+
sample_rate=0.1,
|
|
308
|
+
alert_threshold=5,
|
|
309
|
+
).publish()
|
|
310
|
+
|
|
311
|
+
client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
|
|
312
|
+
```
|
|
313
|
+
|
|
301
314
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
302
315
|
|
|
303
316
|
---
|
|
@@ -330,7 +343,7 @@ Ask a case's question several extra ways each run, LLM-paraphrased server-side,
|
|
|
330
343
|
Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
|
|
331
344
|
|
|
332
345
|
```python
|
|
333
|
-
prompt = client.evaluations.prompts.get("support-agent-system-prompt")
|
|
346
|
+
prompt = client.evaluations.prompts.get("support-agent-system-prompt") # or prompt.id
|
|
334
347
|
# use prompt.text as your own agent's system prompt
|
|
335
348
|
|
|
336
349
|
client.evaluations.run(
|
|
@@ -236,6 +236,19 @@ Per-agent coverage/threshold settings (sample rate, retention, and threshold ove
|
|
|
236
236
|
client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 15000})
|
|
237
237
|
```
|
|
238
238
|
|
|
239
|
+
Self-host also has **online evaluators**: a real LLM judge scoring a sample of live traffic continuously, distinct from a pattern's rule-matching. A score below `alert_threshold` raises a signal the same way a failing pattern does, deduped and triage-ready in `client.monitor.signals`.
|
|
240
|
+
|
|
241
|
+
```python
|
|
242
|
+
evaluator = client.monitor.online_evaluators.builder(
|
|
243
|
+
name="Helpfulness",
|
|
244
|
+
evaluation_settings_id=settings.id,
|
|
245
|
+
sample_rate=0.1,
|
|
246
|
+
alert_threshold=5,
|
|
247
|
+
).publish()
|
|
248
|
+
|
|
249
|
+
client.monitor.online_evaluators.ratings(evaluator.id, window="7d")
|
|
250
|
+
```
|
|
251
|
+
|
|
239
252
|
See **[TRACING.md](TRACING.md)** for the complete Monitor guide.
|
|
240
253
|
|
|
241
254
|
---
|
|
@@ -268,7 +281,7 @@ Ask a case's question several extra ways each run, LLM-paraphrased server-side,
|
|
|
268
281
|
Since AgentX doesn't own your agent's code, `client.evaluations.prompts` lets AgentX become your prompt's *source of truth* instead — the same problem LangSmith's Prompt Hub and Langfuse's Prompt Management solve. Pull a version at runtime, tag your eval runs (or live traces) with it, and let a judge propose a rewrite from your real worst-rated results — a human always has to approve before it publishes:
|
|
269
282
|
|
|
270
283
|
```python
|
|
271
|
-
prompt = client.evaluations.prompts.get("support-agent-system-prompt")
|
|
284
|
+
prompt = client.evaluations.prompts.get("support-agent-system-prompt") # or prompt.id
|
|
272
285
|
# use prompt.text as your own agent's system prompt
|
|
273
286
|
|
|
274
287
|
client.evaluations.run(
|
|
@@ -213,6 +213,8 @@ class EvaluationsClient:
|
|
|
213
213
|
return [Prompt(**p) for p in (data if isinstance(data, list) else data.get("prompts", []))]
|
|
214
214
|
|
|
215
215
|
def get_prompt(self, name: str, version: Optional[int] = None) -> Prompt:
|
|
216
|
+
# `name` doubles as an id: the backend route tries a name match first, then falls back to
|
|
217
|
+
# an id match (see engine's getPromptRowByNameOrId), so callers can pass either.
|
|
216
218
|
params = self._workspace_params() or {}
|
|
217
219
|
if version is not None:
|
|
218
220
|
params = {**params, "version": version}
|
|
@@ -132,7 +132,7 @@ class EvaluationSettings(BaseModel):
|
|
|
132
132
|
# Prompt registry — the external-agent analog to AgentX's native autotune. AgentX doesn't own
|
|
133
133
|
# your agent's code, so instead of branching/applying a config it becomes the prompt's source of
|
|
134
134
|
# truth (same shape as LangSmith's Prompt Hub / Langfuse's Prompt Management): pull a version at
|
|
135
|
-
# runtime with ``client.evaluations.prompts.get(
|
|
135
|
+
# runtime with ``client.evaluations.prompts.get(name_or_id)``, use ``prompt.text`` as your agent's
|
|
136
136
|
# system prompt, and tag your eval runs so the existing version-comparison view can tell you
|
|
137
137
|
# which published version scored higher — see ``client.evaluations.prompts`` docs.
|
|
138
138
|
# ---------------------------------------------------------------------------
|
|
@@ -34,6 +34,9 @@ class PromptClient:
|
|
|
34
34
|
}
|
|
35
35
|
),
|
|
36
36
|
)
|
|
37
|
+
|
|
38
|
+
``get()`` also accepts a prompt's ``id`` in place of its name, e.g.
|
|
39
|
+
``client.evaluations.prompts.get(prompt.id)``.
|
|
37
40
|
"""
|
|
38
41
|
|
|
39
42
|
def __init__(self, client: "EvaluationsClient"):
|
|
@@ -43,6 +46,9 @@ class PromptClient:
|
|
|
43
46
|
return self._client.create_prompt({"name": name, "text": text, "description": description})
|
|
44
47
|
|
|
45
48
|
def get(self, name: str, version: Optional[int] = None) -> Prompt:
|
|
49
|
+
"""Accepts either the prompt's name or its ``id`` (e.g. ``prompt.id`` from an earlier
|
|
50
|
+
``get``/``create`` call), useful for round-tripping an id you already have without a
|
|
51
|
+
second lookup method."""
|
|
46
52
|
return self._client.get_prompt(name, version=version)
|
|
47
53
|
|
|
48
54
|
def list(self) -> List[Prompt]:
|
|
@@ -16,7 +16,6 @@ import inspect
|
|
|
16
16
|
from typing import Any, Callable, Dict, Optional
|
|
17
17
|
|
|
18
18
|
from agentx.tracing.tracer import Tracer
|
|
19
|
-
from agentx.integrations._perf import build_performance_summary
|
|
20
19
|
|
|
21
20
|
|
|
22
21
|
def call_and_trace(
|
|
@@ -82,15 +81,16 @@ def finish_llm_call(
|
|
|
82
81
|
"""
|
|
83
82
|
Close out one raw-client LLM call — shared by the ``on_finish``/exit
|
|
84
83
|
callbacks of every integration that patches a raw provider client
|
|
85
|
-
(``anthropic.py``, ``google_genai.py``, ``openai.py``) rather than a
|
|
84
|
+
(``anthropic.py``, ``google_genai.py``, ``openai.py``, ``litellm.py``) rather than a
|
|
86
85
|
framework-level callback/plugin system.
|
|
87
86
|
|
|
88
|
-
If the call happened inside a ``with tracer.trace(...)`` block,
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
``tracer.
|
|
87
|
+
If the call happened inside a ``with tracer.trace(...)`` block, it becomes that span's own
|
|
88
|
+
real child span (via _record_llm_call) instead of an independent trace — the same "part of a
|
|
89
|
+
multi-call agentic loop" behavior ``anthropic.py`` already had; folded in here so every
|
|
90
|
+
raw-client integration gets it instead of each having to remember to check
|
|
91
|
+
``tracer.current_span`` itself. Otherwise it becomes its own real root span, opened/closed
|
|
92
|
+
directly here (not via ``tracer._send()``) so it still gets a real span_id/session_id and the
|
|
93
|
+
call's exact timing rather than wall-clock "now".
|
|
94
94
|
"""
|
|
95
95
|
latency_ms = int((end_t - start_t) * 1000)
|
|
96
96
|
|
|
@@ -110,32 +110,15 @@ def finish_llm_call(
|
|
|
110
110
|
)
|
|
111
111
|
return
|
|
112
112
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}],
|
|
126
|
-
has_errors=error is not None,
|
|
127
|
-
)
|
|
128
|
-
tracer._send(
|
|
129
|
-
name=name,
|
|
130
|
-
input=input_repr,
|
|
131
|
-
output=output,
|
|
132
|
-
latency_ms=latency_ms,
|
|
133
|
-
error=error,
|
|
134
|
-
framework=framework,
|
|
135
|
-
model=model,
|
|
136
|
-
metadata=metadata,
|
|
137
|
-
session_id=session_id,
|
|
138
|
-
performance_summary=perf,
|
|
139
|
-
input_tokens=input_tokens,
|
|
140
|
-
output_tokens=output_tokens,
|
|
141
|
-
)
|
|
113
|
+
span = tracer.trace(name, metadata=metadata, framework=framework, model=model, session_id=session_id)
|
|
114
|
+
span.__enter__()
|
|
115
|
+
span._start = start_t
|
|
116
|
+
span.input = input_repr
|
|
117
|
+
span.output = output
|
|
118
|
+
if error:
|
|
119
|
+
span.set_error(error)
|
|
120
|
+
if input_tokens:
|
|
121
|
+
span._input_tokens = input_tokens
|
|
122
|
+
if output_tokens:
|
|
123
|
+
span._output_tokens = output_tokens
|
|
124
|
+
span.__exit__(None, None, None)
|
|
@@ -23,7 +23,6 @@ import time
|
|
|
23
23
|
from typing import Any, Dict, List, Optional
|
|
24
24
|
|
|
25
25
|
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
26
|
-
from agentx.integrations._perf import build_performance_summary
|
|
27
26
|
|
|
28
27
|
|
|
29
28
|
def _message_text(message: Any) -> Optional[str]:
|
|
@@ -93,8 +92,6 @@ class AgentXAutoGenObserver:
|
|
|
93
92
|
error = str(exc)
|
|
94
93
|
raise
|
|
95
94
|
finally:
|
|
96
|
-
end_t = time.time()
|
|
97
|
-
latency_ms = int((end_t - start_t) * 1000)
|
|
98
95
|
messages = getattr(result, "messages", None) or []
|
|
99
96
|
execution_steps, tool_call_steps, output_text, input_tokens, output_tokens = self._summarize_messages(
|
|
100
97
|
messages, start_t
|
|
@@ -106,24 +103,23 @@ class AgentXAutoGenObserver:
|
|
|
106
103
|
# normalized .content, regardless of which form was passed in.
|
|
107
104
|
input_text = _message_text(messages[0]) if messages else (task if isinstance(task, str) else None)
|
|
108
105
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
performance_summary=build_performance_summary(
|
|
119
|
-
total_duration_ms=latency_ms,
|
|
106
|
+
# No `return` here — this whole block runs inside the try's `finally`, and an
|
|
107
|
+
# explicit return/break/continue there would silently swallow any exception
|
|
108
|
+
# propagating from agent_or_team.run() above (see crewai.py's kickoff() for the same
|
|
109
|
+
# hazard spelled out in full).
|
|
110
|
+
with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
|
|
111
|
+
span._start = start_t
|
|
112
|
+
if error:
|
|
113
|
+
span.set_error(error)
|
|
114
|
+
span._merge_child_run(
|
|
120
115
|
execution_steps=execution_steps,
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
116
|
+
tool_calls=tool_call_steps,
|
|
117
|
+
input=input_text,
|
|
118
|
+
output=output_text,
|
|
119
|
+
framework="autogen",
|
|
120
|
+
input_tokens=input_tokens or None,
|
|
121
|
+
output_tokens=output_tokens or None,
|
|
122
|
+
)
|
|
127
123
|
|
|
128
124
|
def _summarize_messages(self, messages: List[Any], run_start: float) -> tuple:
|
|
129
125
|
execution_steps: List[Dict[str, Any]] = []
|
|
@@ -22,7 +22,6 @@ import time
|
|
|
22
22
|
from typing import Any, Dict, List, Optional
|
|
23
23
|
|
|
24
24
|
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
25
|
-
from agentx.integrations._perf import build_performance_summary
|
|
26
25
|
|
|
27
26
|
|
|
28
27
|
class AgentXCrewObserver:
|
|
@@ -73,35 +72,30 @@ class AgentXCrewObserver:
|
|
|
73
72
|
output = getattr(result, "raw", None) or _safe_serialize(result)
|
|
74
73
|
|
|
75
74
|
task_outputs = list(getattr(result, "tasks_output", []) or []) if result is not None else []
|
|
76
|
-
tool_calls: List[Dict[str, Any]] = []
|
|
77
75
|
execution_steps: List[Dict[str, Any]] = []
|
|
78
76
|
|
|
79
77
|
if task_timings:
|
|
80
|
-
execution_steps,
|
|
78
|
+
execution_steps, _ = self._build_steps_from_timings(task_timings, task_outputs)
|
|
81
79
|
elif task_outputs:
|
|
82
|
-
execution_steps,
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
tool_calls=tool_calls or None,
|
|
97
|
-
metadata=self._metadata,
|
|
98
|
-
session_id=self._session_id,
|
|
99
|
-
performance_summary=build_performance_summary(
|
|
100
|
-
total_duration_ms=latency_ms,
|
|
80
|
+
execution_steps, _ = self._build_steps_evenly_divided(task_outputs, latency_ms)
|
|
81
|
+
|
|
82
|
+
# Each task becomes its own real child span. tool_calls isn't passed to
|
|
83
|
+
# _merge_child_run here: _build_steps_from_timings/_build_steps_evenly_divided both
|
|
84
|
+
# build it from the exact same per-task loop as execution_steps (no separate timing of
|
|
85
|
+
# its own), so passing both would double-emit each task as two child spans. No
|
|
86
|
+
# `return` here (this whole method body runs inside the try's `finally`) — an
|
|
87
|
+
# explicit return/break/continue in a finally block silently swallows any exception
|
|
88
|
+
# propagating from crew.kickoff() above.
|
|
89
|
+
with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
|
|
90
|
+
span._start = start
|
|
91
|
+
if error:
|
|
92
|
+
span.set_error(error)
|
|
93
|
+
span._merge_child_run(
|
|
101
94
|
execution_steps=execution_steps,
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
95
|
+
input=_safe_serialize(inputs) if inputs else None,
|
|
96
|
+
output=output,
|
|
97
|
+
framework="crewai",
|
|
98
|
+
)
|
|
105
99
|
|
|
106
100
|
def _start_task_timing_capture(self):
|
|
107
101
|
"""
|
|
@@ -24,7 +24,6 @@ import time
|
|
|
24
24
|
from typing import Any, Dict, List, Optional
|
|
25
25
|
|
|
26
26
|
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
27
|
-
from agentx.integrations._perf import build_performance_summary
|
|
28
27
|
|
|
29
28
|
try:
|
|
30
29
|
from google.adk.plugins.base_plugin import BasePlugin
|
|
@@ -82,7 +81,8 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
82
81
|
Google ADK plugin that sends one AgentX trace per runner invocation.
|
|
83
82
|
|
|
84
83
|
Captures input (user message), output (final model reply), model name,
|
|
85
|
-
tool calls, latency,
|
|
84
|
+
tool calls, and latency, each model/tool call as its own real child span of the
|
|
85
|
+
invocation's root, via the ADK plugin callbacks.
|
|
86
86
|
|
|
87
87
|
Register via the ``plugins`` list when constructing the ADK ``Runner``.
|
|
88
88
|
"""
|
|
@@ -128,16 +128,21 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
128
128
|
async def before_run_callback(self, *, invocation_context: Any) -> None:
|
|
129
129
|
inv_id = invocation_context.invocation_id
|
|
130
130
|
agent_name = getattr(invocation_context.agent, "name", None) or self._agent_name
|
|
131
|
+
# Held directly (not relied on via tracer.current_span) — ADK callbacks for one
|
|
132
|
+
# invocation can interleave with callbacks for a different concurrent invocation on the
|
|
133
|
+
# same thread/task, so state["root_span"] (keyed by invocation_id, same as everything
|
|
134
|
+
# else here) is the reliable way to address the right parent.
|
|
135
|
+
root_span = self._tracer.trace(
|
|
136
|
+
agent_name, framework="google-adk", metadata=self._metadata, session_id=self._session_id
|
|
137
|
+
)
|
|
138
|
+
root_span.__enter__()
|
|
131
139
|
self._runs[inv_id] = {
|
|
132
|
-
"
|
|
133
|
-
"
|
|
140
|
+
"root_span": root_span,
|
|
141
|
+
"llm_call_count": 0,
|
|
134
142
|
"input": self._pending_inputs.pop(inv_id, None),
|
|
135
143
|
"output": None,
|
|
136
144
|
"model": None,
|
|
137
|
-
"tool_calls": [],
|
|
138
145
|
"error": None,
|
|
139
|
-
"execution_steps": [],
|
|
140
|
-
"perf_tool_calls": [],
|
|
141
146
|
"input_tokens": 0,
|
|
142
147
|
"output_tokens": 0,
|
|
143
148
|
}
|
|
@@ -147,27 +152,18 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
147
152
|
state = self._runs.pop(inv_id, None)
|
|
148
153
|
if state is None:
|
|
149
154
|
return
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
framework="google-adk",
|
|
163
|
-
model=state["model"],
|
|
164
|
-
tool_calls=state["tool_calls"] or None,
|
|
165
|
-
metadata=self._metadata,
|
|
166
|
-
session_id=self._session_id,
|
|
167
|
-
performance_summary=perf,
|
|
168
|
-
input_tokens=state["input_tokens"] or None,
|
|
169
|
-
output_tokens=state["output_tokens"] or None,
|
|
170
|
-
)
|
|
155
|
+
# This invocation's own detail already went out as child-span rows via child_span() in
|
|
156
|
+
# the model/tool callbacks below — this just closes the root with a summary
|
|
157
|
+
# input/output/model/error/tokens.
|
|
158
|
+
root_span = state["root_span"]
|
|
159
|
+
root_span.input = state["input"]
|
|
160
|
+
root_span.output = state["output"]
|
|
161
|
+
root_span._captured_model = state["model"]
|
|
162
|
+
root_span._input_tokens = state["input_tokens"]
|
|
163
|
+
root_span._output_tokens = state["output_tokens"]
|
|
164
|
+
if state["error"]:
|
|
165
|
+
root_span.set_error(state["error"])
|
|
166
|
+
root_span.__exit__(None, None, None)
|
|
171
167
|
|
|
172
168
|
# ------------------------------------------------------------------
|
|
173
169
|
# Model callbacks — capture model name, output, and LLM step timing
|
|
@@ -223,20 +219,19 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
223
219
|
if call_output_tokens is not None:
|
|
224
220
|
state["output_tokens"] += int(call_output_tokens)
|
|
225
221
|
|
|
226
|
-
# Execution step
|
|
222
|
+
# Execution step — a real child span of this invocation's root.
|
|
227
223
|
if start_t is not None:
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
"
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
})
|
|
224
|
+
state["llm_call_count"] += 1
|
|
225
|
+
state["root_span"].child_span(
|
|
226
|
+
f"LLM Call {state['llm_call_count']}",
|
|
227
|
+
start_time=start_t,
|
|
228
|
+
end_time=end_t,
|
|
229
|
+
input=call_start.get("input") if call_start else None,
|
|
230
|
+
output=text,
|
|
231
|
+
model=call_start.get("model") if call_start else None,
|
|
232
|
+
input_tokens=call_input_tokens,
|
|
233
|
+
output_tokens=call_output_tokens,
|
|
234
|
+
)
|
|
240
235
|
|
|
241
236
|
async def on_model_error_callback(
|
|
242
237
|
self, *, callback_context: Any, llm_request: Any, error: Exception
|
|
@@ -257,16 +252,16 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
257
252
|
state["error"] = str(error)
|
|
258
253
|
|
|
259
254
|
if start_t is not None:
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
"
|
|
266
|
-
"
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
255
|
+
state["llm_call_count"] += 1
|
|
256
|
+
state["root_span"].child_span(
|
|
257
|
+
f"LLM Call {state['llm_call_count']}",
|
|
258
|
+
start_time=start_t,
|
|
259
|
+
end_time=end_t,
|
|
260
|
+
input=call_start.get("input") if call_start else None,
|
|
261
|
+
output=f"ERROR: {error}",
|
|
262
|
+
model=call_start.get("model") if call_start else None,
|
|
263
|
+
error=str(error),
|
|
264
|
+
)
|
|
270
265
|
|
|
271
266
|
# ------------------------------------------------------------------
|
|
272
267
|
# Tool callbacks
|
|
@@ -294,23 +289,9 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
294
289
|
tool_name = getattr(tool, "name", "unknown")
|
|
295
290
|
tool_input = _safe_serialize(tool_args)
|
|
296
291
|
tool_output = str(result) if result is not None else None
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
"output": tool_output,
|
|
301
|
-
}
|
|
302
|
-
if start_t is not None:
|
|
303
|
-
tool_call["latency_ms"] = max(0, int((end_t - start_t) * 1000))
|
|
304
|
-
state["tool_calls"].append(tool_call)
|
|
305
|
-
if start_t is not None:
|
|
306
|
-
state["perf_tool_calls"].append({
|
|
307
|
-
"name": tool_name,
|
|
308
|
-
"duration_ms": (end_t - start_t) * 1000,
|
|
309
|
-
"start_time": start_t,
|
|
310
|
-
"end_time": end_t,
|
|
311
|
-
"input": tool_input,
|
|
312
|
-
"output": tool_output,
|
|
313
|
-
})
|
|
292
|
+
state["root_span"].child_span(
|
|
293
|
+
tool_name, start_time=start_t, end_time=end_t, input=tool_input, output=tool_output
|
|
294
|
+
)
|
|
314
295
|
|
|
315
296
|
async def on_tool_error_callback(
|
|
316
297
|
self,
|
|
@@ -329,20 +310,6 @@ class AgentXADKPlugin(BasePlugin):
|
|
|
329
310
|
tool_name = getattr(tool, "name", "unknown")
|
|
330
311
|
tool_input = _safe_serialize(tool_args)
|
|
331
312
|
tool_output = f"ERROR: {error}"
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
"output": tool_output,
|
|
336
|
-
}
|
|
337
|
-
if start_t is not None:
|
|
338
|
-
tool_call["latency_ms"] = max(0, int((end_t - start_t) * 1000))
|
|
339
|
-
state["tool_calls"].append(tool_call)
|
|
340
|
-
if start_t is not None:
|
|
341
|
-
state["perf_tool_calls"].append({
|
|
342
|
-
"name": tool_name,
|
|
343
|
-
"duration_ms": (end_t - start_t) * 1000,
|
|
344
|
-
"start_time": start_t,
|
|
345
|
-
"end_time": end_t,
|
|
346
|
-
"input": tool_input,
|
|
347
|
-
"output": tool_output,
|
|
348
|
-
})
|
|
313
|
+
state["root_span"].child_span(
|
|
314
|
+
tool_name, start_time=start_t, end_time=end_t, input=tool_input, output=tool_output, error=str(error)
|
|
315
|
+
)
|
|
@@ -26,7 +26,6 @@ from typing import Any, Dict, List, Optional, Union
|
|
|
26
26
|
from uuid import UUID
|
|
27
27
|
|
|
28
28
|
from agentx.tracing.tracer import Tracer, _safe_serialize
|
|
29
|
-
from agentx.integrations._perf import build_performance_summary
|
|
30
29
|
|
|
31
30
|
try:
|
|
32
31
|
from langchain_core.callbacks.base import BaseCallbackHandler
|
|
@@ -329,7 +328,6 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
329
328
|
"tool_calls": [],
|
|
330
329
|
"model": None,
|
|
331
330
|
"execution_steps": [],
|
|
332
|
-
"perf_tool_calls": [],
|
|
333
331
|
"retrieval_steps": pending,
|
|
334
332
|
"input_tokens": 0,
|
|
335
333
|
"output_tokens": 0,
|
|
@@ -355,7 +353,6 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
355
353
|
state = self._runs.pop(run_id, None)
|
|
356
354
|
if state is None:
|
|
357
355
|
return
|
|
358
|
-
latency_ms = int((time.time() - state["start"]) * 1000)
|
|
359
356
|
output = _extract_output(outputs)
|
|
360
357
|
# Each tool_call dict already carries its own start_time/end_time (set in
|
|
361
358
|
# on_tool_end/on_tool_error), so no re-pairing against perf_tool_calls by
|
|
@@ -383,26 +380,24 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
383
380
|
output_tokens=state["output_tokens"] or None,
|
|
384
381
|
)
|
|
385
382
|
else:
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
output_tokens=state["output_tokens"] or None,
|
|
405
|
-
)
|
|
383
|
+
# Standalone usage (no enclosing `with tracer.trace()`): open a real root span for
|
|
384
|
+
# this chain invocation and let _merge_child_run explode its accumulated
|
|
385
|
+
# execution_steps/tool_calls/retrieval_steps into real child-span rows.
|
|
386
|
+
with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
|
|
387
|
+
# __enter__ just set _start to "now" — overridden to the chain's real start time,
|
|
388
|
+
# see llamaindex.py's _send_trace for the identical fix and full rationale.
|
|
389
|
+
span._start = state["start"]
|
|
390
|
+
span._merge_child_run(
|
|
391
|
+
execution_steps=state["execution_steps"],
|
|
392
|
+
tool_calls=tool_calls,
|
|
393
|
+
retrieval_steps=state["retrieval_steps"],
|
|
394
|
+
input=state["input"],
|
|
395
|
+
output=output,
|
|
396
|
+
model=state.get("model"),
|
|
397
|
+
framework="langchain",
|
|
398
|
+
input_tokens=state["input_tokens"] or None,
|
|
399
|
+
output_tokens=state["output_tokens"] or None,
|
|
400
|
+
)
|
|
406
401
|
|
|
407
402
|
def on_chain_error(
|
|
408
403
|
self,
|
|
@@ -420,7 +415,6 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
420
415
|
state = self._runs.pop(run_id, None)
|
|
421
416
|
if state is None:
|
|
422
417
|
return
|
|
423
|
-
latency_ms = int((time.time() - state["start"]) * 1000)
|
|
424
418
|
|
|
425
419
|
active_span = self._tracer.current_span
|
|
426
420
|
if active_span is not None:
|
|
@@ -436,27 +430,20 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
436
430
|
output_tokens=state["output_tokens"] or None,
|
|
437
431
|
)
|
|
438
432
|
else:
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
tool_calls=state["tool_calls"] or None,
|
|
454
|
-
metadata=self._metadata,
|
|
455
|
-
session_id=self._session_id,
|
|
456
|
-
performance_summary=perf,
|
|
457
|
-
input_tokens=state["input_tokens"] or None,
|
|
458
|
-
output_tokens=state["output_tokens"] or None,
|
|
459
|
-
)
|
|
433
|
+
# See on_chain_end's matching branch — same standalone-usage handling.
|
|
434
|
+
with self._tracer.trace(self._name, metadata=self._metadata, session_id=self._session_id) as span:
|
|
435
|
+
span._start = state["start"]
|
|
436
|
+
span.set_error(str(error))
|
|
437
|
+
span._merge_child_run(
|
|
438
|
+
execution_steps=state["execution_steps"],
|
|
439
|
+
tool_calls=state["tool_calls"],
|
|
440
|
+
retrieval_steps=state["retrieval_steps"],
|
|
441
|
+
input=state["input"],
|
|
442
|
+
model=state.get("model"),
|
|
443
|
+
framework="langchain",
|
|
444
|
+
input_tokens=state["input_tokens"] or None,
|
|
445
|
+
output_tokens=state["output_tokens"] or None,
|
|
446
|
+
)
|
|
460
447
|
|
|
461
448
|
# ------------------------------------------------------------------
|
|
462
449
|
# LLM lifecycle
|
|
@@ -620,14 +607,6 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
620
607
|
with self._state_lock:
|
|
621
608
|
if top and top in self._runs:
|
|
622
609
|
self._runs[top]["tool_calls"].append(tool_call)
|
|
623
|
-
self._runs[top]["perf_tool_calls"].append({
|
|
624
|
-
"name": state["tool_name"],
|
|
625
|
-
"duration_ms": (end_t - start_t) * 1000,
|
|
626
|
-
"start_time": start_t,
|
|
627
|
-
"end_time": end_t,
|
|
628
|
-
"input": state["tool_input"],
|
|
629
|
-
"output": tool_call["output"],
|
|
630
|
-
})
|
|
631
610
|
|
|
632
611
|
def on_tool_error(
|
|
633
612
|
self,
|
|
@@ -656,14 +635,6 @@ class AgentXCallbackHandler(BaseCallbackHandler):
|
|
|
656
635
|
with self._state_lock:
|
|
657
636
|
if top and top in self._runs:
|
|
658
637
|
self._runs[top]["tool_calls"].append(tool_call)
|
|
659
|
-
self._runs[top]["perf_tool_calls"].append({
|
|
660
|
-
"name": state.get("tool_name", "unknown"),
|
|
661
|
-
"duration_ms": (end_t - start_t) * 1000,
|
|
662
|
-
"start_time": start_t,
|
|
663
|
-
"end_time": end_t,
|
|
664
|
-
"input": tool_call["input"],
|
|
665
|
-
"output": tool_call["output"],
|
|
666
|
-
})
|
|
667
638
|
|
|
668
639
|
# ------------------------------------------------------------------
|
|
669
640
|
# Retriever lifecycle
|