fableforge-agent-telemetry 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_telemetry/__init__.py +7 -0
- agent_telemetry/cli.py +212 -0
- agent_telemetry/collector.py +412 -0
- agent_telemetry/dashboard.py +462 -0
- agent_telemetry/error_tracker.py +149 -0
- agent_telemetry/models.py +110 -0
- agent_telemetry/storage.py +501 -0
- agent_telemetry/token_tracker.py +195 -0
- fableforge_agent_telemetry-0.1.0.dist-info/METADATA +21 -0
- fableforge_agent_telemetry-0.1.0.dist-info/RECORD +14 -0
- fableforge_agent_telemetry-0.1.0.dist-info/WHEEL +5 -0
- fableforge_agent_telemetry-0.1.0.dist-info/entry_points.txt +2 -0
- fableforge_agent_telemetry-0.1.0.dist-info/licenses/LICENSE +21 -0
- fableforge_agent_telemetry-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
"""FastAPI dashboard with Plotly charts for agent telemetry data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
from fastapi import FastAPI, Request
|
|
10
|
+
from fastapi.responses import HTMLResponse
|
|
11
|
+
|
|
12
|
+
from agent_telemetry.collector import calculate_metrics, ingest_trace
|
|
13
|
+
from agent_telemetry.error_tracker import detect_errors, generate_error_report
|
|
14
|
+
from agent_telemetry.models import Span
|
|
15
|
+
from agent_telemetry.storage import TelemetryStorage
|
|
16
|
+
from agent_telemetry.token_tracker import estimate_cost
|
|
17
|
+
|
|
18
|
+
app = FastAPI(title="AgentTelemetry", version="0.1.0")
|
|
19
|
+
|
|
20
|
+
_storage: Optional[TelemetryStorage] = None
|
|
21
|
+
_spans_cache: dict[str, list[Span]] = {}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _plotly_cdn() -> str:
|
|
25
|
+
return """<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script>"""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _base_html(title: str, body: str) -> str:
|
|
29
|
+
return f"""<!DOCTYPE html>
|
|
30
|
+
<html lang="en">
|
|
31
|
+
<head>
|
|
32
|
+
<meta charset="utf-8">
|
|
33
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
34
|
+
<title>{html.escape(title)} — AgentTelemetry</title>
|
|
35
|
+
<style>
|
|
36
|
+
:root {{ --bg: #0f172a; --surface: #1e293b; --text: #e2e8f0; --accent: #38bdf8; --red: #f87171; --green: #4ade80; --yellow: #fbbf24; }}
|
|
37
|
+
body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 0; padding: 1rem; background: var(--bg); color: var(--text); }}
|
|
38
|
+
a {{ color: var(--accent); text-decoration: none; }}
|
|
39
|
+
a:hover {{ text-decoration: underline; }}
|
|
40
|
+
.card {{ background: var(--surface); border-radius: 0.5rem; padding: 1.25rem; margin-bottom: 1rem; }}
|
|
41
|
+
h1, h2, h3 {{ margin-top: 0; }}
|
|
42
|
+
table {{ width: 100%; border-collapse: collapse; }}
|
|
43
|
+
th, td {{ text-align: left; padding: 0.5rem 0.75rem; border-bottom: 1px solid #334155; }}
|
|
44
|
+
th {{ color: var(--accent); font-weight: 600; }}
|
|
45
|
+
.metric {{ display: inline-block; background: var(--surface); border-radius: 0.5rem; padding: 1rem 1.5rem; margin: 0.25rem; min-width: 140px; }}
|
|
46
|
+
.metric .value {{ font-size: 1.75rem; font-weight: 700; }}
|
|
47
|
+
.metric .label {{ font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; color: #94a3b8; }}
|
|
48
|
+
nav {{ margin-bottom: 1rem; }}
|
|
49
|
+
nav a {{ margin-right: 1rem; }}
|
|
50
|
+
.chart {{ width: 100%; height: 400px; }}
|
|
51
|
+
.badge {{ display: inline-block; padding: 0.125rem 0.5rem; border-radius: 0.25rem; font-size: 0.75rem; font-weight: 600; }}
|
|
52
|
+
.badge-ok {{ background: #166534; color: var(--green); }}
|
|
53
|
+
.badge-error {{ background: #7f1d1d; color: var(--red); }}
|
|
54
|
+
.badge-timeout {{ background: #78350f; color: var(--yellow); }}
|
|
55
|
+
</style>
|
|
56
|
+
{_plotly_cdn()}
|
|
57
|
+
</head>
|
|
58
|
+
<body>
|
|
59
|
+
<nav>
|
|
60
|
+
<a href="/dashboard">Dashboard</a>
|
|
61
|
+
<a href="/cost/report">Cost Report</a>
|
|
62
|
+
<a href="/errors/report">Error Report</a>
|
|
63
|
+
</nav>
|
|
64
|
+
{body}
|
|
65
|
+
</body>
|
|
66
|
+
</html>"""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _init_storage() -> TelemetryStorage:
|
|
70
|
+
global _storage
|
|
71
|
+
if _storage is None:
|
|
72
|
+
_storage = TelemetryStorage()
|
|
73
|
+
return _storage
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _load_spans(session_id: str | None = None) -> list[Span]:
|
|
77
|
+
storage = _init_storage()
|
|
78
|
+
if session_id:
|
|
79
|
+
if session_id in _spans_cache:
|
|
80
|
+
return _spans_cache[session_id]
|
|
81
|
+
spans = storage.query_spans(session_id=session_id, limit=100000)
|
|
82
|
+
_spans_cache[session_id] = spans
|
|
83
|
+
return spans
|
|
84
|
+
return []
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _make_timeline_chart(spans: list[Span], div_id: str = "timeline") -> str:
|
|
88
|
+
if not spans:
|
|
89
|
+
return f'<div id="{div_id}"></div><p>No data.</p>'
|
|
90
|
+
|
|
91
|
+
timestamps = [s.timestamp.isoformat() if s.timestamp else "" for s in spans]
|
|
92
|
+
durations = [s.duration_ms for s in spans]
|
|
93
|
+
tools = [s.tool_name for s in spans]
|
|
94
|
+
colors = {"ok": "#4ade80", "error": "#f87171", "timeout": "#fbbf24"}
|
|
95
|
+
bar_colors = [colors.get(s.status.value, "#94a3b8") for s in spans]
|
|
96
|
+
|
|
97
|
+
return f"""
|
|
98
|
+
<div id="{div_id}" class="chart"></div>
|
|
99
|
+
<script>
|
|
100
|
+
Plotly.newPlot('{div_id}', [{{
|
|
101
|
+
x: {timestamps},
|
|
102
|
+
y: {durations},
|
|
103
|
+
type: 'bar',
|
|
104
|
+
marker: {{ color: {bar_colors} }},
|
|
105
|
+
text: {tools},
|
|
106
|
+
hovertemplate: '%{{text}}<br>%{{y}} ms<extra></extra>'
|
|
107
|
+
}}], {{
|
|
108
|
+
title: 'Tool Call Timeline',
|
|
109
|
+
xaxis: {{ title: 'Time' }},
|
|
110
|
+
yaxis: {{ title: 'Duration (ms)', type: 'log' }},
|
|
111
|
+
paper_bgcolor: '#0f172a',
|
|
112
|
+
plot_bgcolor: '#1e293b',
|
|
113
|
+
font: {{ color: '#e2e8f0' }},
|
|
114
|
+
margin: {{ t: 40 }}
|
|
115
|
+
}});
|
|
116
|
+
</script>"""
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _make_heatmap_chart(spans: list[Span], div_id: str = "heatmap") -> str:
|
|
120
|
+
if not spans:
|
|
121
|
+
return f'<div id="{div_id}"></div><p>No data.</p>'
|
|
122
|
+
|
|
123
|
+
tool_names: list[str] = sorted(set(s.tool_name for s in spans))
|
|
124
|
+
hours: list[str] = sorted(set(
|
|
125
|
+
(s.timestamp.strftime("%Y-%m-%d %H:00") if s.timestamp else "unknown")
|
|
126
|
+
for s in spans
|
|
127
|
+
))
|
|
128
|
+
|
|
129
|
+
z: list[list[int]] = []
|
|
130
|
+
for tool in tool_names:
|
|
131
|
+
row = []
|
|
132
|
+
for hour in hours:
|
|
133
|
+
count = sum(
|
|
134
|
+
1 for s in spans
|
|
135
|
+
if s.tool_name == tool
|
|
136
|
+
and (s.timestamp.strftime("%Y-%m-%d %H:00") if s.timestamp else "unknown") == hour
|
|
137
|
+
)
|
|
138
|
+
row.append(count)
|
|
139
|
+
z.append(row)
|
|
140
|
+
|
|
141
|
+
return f"""
|
|
142
|
+
<div id="{div_id}" class="chart"></div>
|
|
143
|
+
<script>
|
|
144
|
+
Plotly.newPlot('{div_id}', [{{
|
|
145
|
+
z: {z},
|
|
146
|
+
x: {hours},
|
|
147
|
+
y: {tool_names},
|
|
148
|
+
type: 'heatmap',
|
|
149
|
+
colorscale: 'Viridis',
|
|
150
|
+
hovertemplate: '%{{y}} at %{{x}}<br>%{{z}} calls<extra></extra>'
|
|
151
|
+
}}], {{
|
|
152
|
+
title: 'Tool Usage Heatmap',
|
|
153
|
+
xaxis: {{ title: 'Time' }},
|
|
154
|
+
yaxis: {{ title: 'Tool' }},
|
|
155
|
+
paper_bgcolor: '#0f172a',
|
|
156
|
+
plot_bgcolor: '#1e293b',
|
|
157
|
+
font: {{ color: '#e2e8f0' }},
|
|
158
|
+
margin: {{ t: 40 }}
|
|
159
|
+
}});
|
|
160
|
+
</script>"""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _make_cost_chart(spans: list[Span], div_id: str = "cost_chart") -> str:
|
|
164
|
+
if not spans:
|
|
165
|
+
return f'<div id="{div_id}"></div><p>No data.</p>'
|
|
166
|
+
|
|
167
|
+
by_tool: dict[str, float] = {}
|
|
168
|
+
for s in spans:
|
|
169
|
+
by_tool[s.tool_name] = by_tool.get(s.tool_name, 0.0) + s.cost_usd
|
|
170
|
+
|
|
171
|
+
tools = list(by_tool.keys())
|
|
172
|
+
costs = list(by_tool.values())
|
|
173
|
+
|
|
174
|
+
return f"""
|
|
175
|
+
<div id="{div_id}" class="chart"></div>
|
|
176
|
+
<script>
|
|
177
|
+
Plotly.newPlot('{div_id}', [{{
|
|
178
|
+
labels: {tools},
|
|
179
|
+
values: {costs},
|
|
180
|
+
type: 'pie',
|
|
181
|
+
hole: 0.4,
|
|
182
|
+
textinfo: 'label+percent',
|
|
183
|
+
textfont: {{ color: '#e2e8f0' }},
|
|
184
|
+
marker: {{ colors: ['#38bdf8','#4ade80','#fbbf24','#f87171','#a78bfa','#fb923c','#2dd4bf','#e879f9'] }}
|
|
185
|
+
}}], {{
|
|
186
|
+
title: 'Cost by Tool',
|
|
187
|
+
paper_bgcolor: '#0f172a',
|
|
188
|
+
plot_bgcolor: '#1e293b',
|
|
189
|
+
font: {{ color: '#e2e8f0' }},
|
|
190
|
+
margin: {{ t: 40 }}
|
|
191
|
+
}});
|
|
192
|
+
</script>"""
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _make_error_chart(session_id: str, div_id: str = "error_chart") -> str:
|
|
196
|
+
from agent_telemetry.models import Span as S
|
|
197
|
+
spans = _load_spans(session_id)
|
|
198
|
+
errors = detect_errors(spans)
|
|
199
|
+
|
|
200
|
+
by_type: dict[str, int] = {}
|
|
201
|
+
for e in errors:
|
|
202
|
+
by_type[e.error_type] = by_type.get(e.error_type, 0) + 1
|
|
203
|
+
|
|
204
|
+
if not by_type:
|
|
205
|
+
return f'<div id="{div_id}"></div><p>No errors found.</p>'
|
|
206
|
+
|
|
207
|
+
types = list(by_type.keys())
|
|
208
|
+
counts = list(by_type.values())
|
|
209
|
+
|
|
210
|
+
return f"""
|
|
211
|
+
<div id="{div_id}" class="chart"></div>
|
|
212
|
+
<script>
|
|
213
|
+
Plotly.newPlot('{div_id}', [{{
|
|
214
|
+
x: {types},
|
|
215
|
+
y: {counts},
|
|
216
|
+
type: 'bar',
|
|
217
|
+
marker: {{ color: '#f87171' }}
|
|
218
|
+
}}], {{
|
|
219
|
+
title: 'Errors by Type',
|
|
220
|
+
xaxis: {{ title: 'Error Type' }},
|
|
221
|
+
yaxis: {{ title: 'Count' }},
|
|
222
|
+
paper_bgcolor: '#0f172a',
|
|
223
|
+
plot_bgcolor: '#1e293b',
|
|
224
|
+
font: {{ color: '#e2e8f0' }},
|
|
225
|
+
margin: {{ t: 40 }}
|
|
226
|
+
}});
|
|
227
|
+
</script>"""
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
@app.get("/dashboard", response_class=HTMLResponse)
|
|
231
|
+
async def dashboard(request: Request):
|
|
232
|
+
storage = _init_storage()
|
|
233
|
+
sessions = storage.list_sessions()
|
|
234
|
+
|
|
235
|
+
session_cards = ""
|
|
236
|
+
for sid in sessions:
|
|
237
|
+
metrics = storage.get_session_metrics(sid)
|
|
238
|
+
if metrics:
|
|
239
|
+
session_cards += f"""
|
|
240
|
+
<div class="card">
|
|
241
|
+
<h3><a href="/sessions/{sid}">{html.escape(sid)}</a></h3>
|
|
242
|
+
<div class="metric"><div class="label">Tokens</div><div class="value">{metrics.total_tokens:,}</div></div>
|
|
243
|
+
<div class="metric"><div class="label">Cost</div><div class="value">${metrics.total_cost:.4f}</div></div>
|
|
244
|
+
<div class="metric"><div class="label">Tool Calls</div><div class="value">{metrics.tool_calls}</div></div>
|
|
245
|
+
<div class="metric"><div class="label">Errors</div><div class="value">{metrics.error_count}</div></div>
|
|
246
|
+
<div class="metric"><div class="label">Duration</div><div class="value">{metrics.duration_seconds:.1f}s</div></div>
|
|
247
|
+
</div>"""
|
|
248
|
+
|
|
249
|
+
if not session_cards:
|
|
250
|
+
session_cards = '<div class="card"><p>No sessions found. Use <code>agenttelemetry analyze <trace.jsonl></code> to ingest traces.</p></div>'
|
|
251
|
+
|
|
252
|
+
body = f"""
|
|
253
|
+
<h1>AgentTelemetry Dashboard</h1>
|
|
254
|
+
{session_cards}
|
|
255
|
+
"""
|
|
256
|
+
return HTMLResponse(_base_html("Dashboard", body))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@app.get("/sessions/{session_id}", response_class=HTMLResponse)
|
|
260
|
+
async def session_detail(request: Request, session_id: str):
|
|
261
|
+
storage = _init_storage()
|
|
262
|
+
metrics = storage.get_session_metrics(session_id)
|
|
263
|
+
spans = _load_spans(session_id)
|
|
264
|
+
|
|
265
|
+
if not metrics:
|
|
266
|
+
body = f'<div class="card"><p>Session <code>{html.escape(session_id)}</code> not found.</p></div>'
|
|
267
|
+
return HTMLResponse(_base_html("Session Not Found", body))
|
|
268
|
+
|
|
269
|
+
spans_rows = ""
|
|
270
|
+
for s in spans[:100]:
|
|
271
|
+
badge_class = f"badge-{s.status.value}"
|
|
272
|
+
spans_rows += f"""
|
|
273
|
+
<tr>
|
|
274
|
+
<td>{html.escape(s.tool_name)}</td>
|
|
275
|
+
<td>{s.input_tokens:,}</td>
|
|
276
|
+
<td>{s.output_tokens:,}</td>
|
|
277
|
+
<td>{s.duration_ms:.0f}ms</td>
|
|
278
|
+
<td>${s.cost_usd:.6f}</td>
|
|
279
|
+
<td><span class="badge {badge_class}">{s.status.value}</span></td>
|
|
280
|
+
<td>{html.escape(s.error or "")[:80]}</td>
|
|
281
|
+
</tr>"""
|
|
282
|
+
|
|
283
|
+
body = f"""
|
|
284
|
+
<h1>Session: {html.escape(session_id)}</h1>
|
|
285
|
+
<div class="card">
|
|
286
|
+
<div class="metric"><div class="label">Total Tokens</div><div class="value">{metrics.total_tokens:,}</div></div>
|
|
287
|
+
<div class="metric"><div class="label">Total Cost</div><div class="value">${metrics.total_cost:.4f}</div></div>
|
|
288
|
+
<div class="metric"><div class="label">Tool Calls</div><div class="value">{metrics.tool_calls}</div></div>
|
|
289
|
+
<div class="metric"><div class="label">Errors</div><div class="value">{metrics.error_count}</div></div>
|
|
290
|
+
<div class="metric"><div class="label">Cache Hit Rate</div><div class="value">{metrics.cache_hit_rate:.1%}</div></div>
|
|
291
|
+
<div class="metric"><div class="label">P50 Duration</div><div class="value">{metrics.p50_duration_ms:.0f}ms</div></div>
|
|
292
|
+
<div class="metric"><div class="label">P95 Duration</div><div class="value">{metrics.p95_duration_ms:.0f}ms</div></div>
|
|
293
|
+
<div class="metric"><div class="label">P99 Duration</div><div class="value">{metrics.p99_duration_ms:.0f}ms</div></div>
|
|
294
|
+
</div>
|
|
295
|
+
<div class="card">
|
|
296
|
+
<h3>Tool Calls</h3>
|
|
297
|
+
<table>
|
|
298
|
+
<tr><th>Tool</th><th>Input</th><th>Output</th><th>Duration</th><th>Cost</th><th>Status</th><th>Error</th></tr>
|
|
299
|
+
{spans_rows}
|
|
300
|
+
</table>
|
|
301
|
+
</div>
|
|
302
|
+
<div class="card">
|
|
303
|
+
<a href="/sessions/{session_id}/timeline">Timeline</a> |
|
|
304
|
+
<a href="/sessions/{session_id}/heatmap">Heatmap</a> |
|
|
305
|
+
<a href="/cost/report?session_id={session_id}">Cost Report</a>
|
|
306
|
+
</div>
|
|
307
|
+
"""
|
|
308
|
+
return HTMLResponse(_base_html(f"Session {session_id}", body))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@app.get("/sessions/{session_id}/timeline", response_class=HTMLResponse)
|
|
312
|
+
async def session_timeline(request: Request, session_id: str):
|
|
313
|
+
spans = _load_spans(session_id)
|
|
314
|
+
chart = _make_timeline_chart(spans)
|
|
315
|
+
cost_chart = _make_cost_chart(spans)
|
|
316
|
+
|
|
317
|
+
body = f"""
|
|
318
|
+
<h1>Timeline: {html.escape(session_id)}</h1>
|
|
319
|
+
<div class="card">{chart}</div>
|
|
320
|
+
<div class="card">{cost_chart}</div>
|
|
321
|
+
"""
|
|
322
|
+
return HTMLResponse(_base_html(f"Timeline — {session_id}", body))
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@app.get("/sessions/{session_id}/heatmap", response_class=HTMLResponse)
|
|
326
|
+
async def session_heatmap(request: Request, session_id: str):
|
|
327
|
+
spans = _load_spans(session_id)
|
|
328
|
+
chart = _make_heatmap_chart(spans)
|
|
329
|
+
|
|
330
|
+
body = f"""
|
|
331
|
+
<h1>Heatmap: {html.escape(session_id)}</h1>
|
|
332
|
+
<div class="card">{chart}</div>
|
|
333
|
+
"""
|
|
334
|
+
return HTMLResponse(_base_html(f"Heatmap — {session_id}", body))
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
@app.get("/cost/report", response_class=HTMLResponse)
|
|
338
|
+
async def cost_report(request: Request, session_id: Optional[str] = None):
|
|
339
|
+
storage = _init_storage()
|
|
340
|
+
spans = _load_spans(session_id) if session_id else []
|
|
341
|
+
|
|
342
|
+
if not spans:
|
|
343
|
+
all_tools = storage.aggregate_tool_metrics()
|
|
344
|
+
if not all_tools:
|
|
345
|
+
body = '<div class="card"><p>No cost data available.</p></div>'
|
|
346
|
+
return HTMLResponse(_base_html("Cost Report", body))
|
|
347
|
+
|
|
348
|
+
rows = ""
|
|
349
|
+
total_cost = 0.0
|
|
350
|
+
for t in all_tools:
|
|
351
|
+
total_cost += t.total_cost_usd
|
|
352
|
+
rows += f"""
|
|
353
|
+
<tr>
|
|
354
|
+
<td>{html.escape(t.tool_name)}</td>
|
|
355
|
+
<td>{t.call_count}</td>
|
|
356
|
+
<td>${t.total_cost_usd:.6f}</td>
|
|
357
|
+
<td>{t.total_input_tokens:,}</td>
|
|
358
|
+
<td>{t.total_output_tokens:,}</td>
|
|
359
|
+
</tr>"""
|
|
360
|
+
|
|
361
|
+
body = f"""
|
|
362
|
+
<h1>Cost Report — All Sessions</h1>
|
|
363
|
+
<div class="card">
|
|
364
|
+
<div class="metric"><div class="label">Total Cost</div><div class="value">${total_cost:.4f}</div></div>
|
|
365
|
+
</div>
|
|
366
|
+
<div class="card">
|
|
367
|
+
<table>
|
|
368
|
+
<tr><th>Tool</th><th>Calls</th><th>Cost</th><th>Input Tokens</th><th>Output Tokens</th></tr>
|
|
369
|
+
{rows}
|
|
370
|
+
</table>
|
|
371
|
+
</div>
|
|
372
|
+
"""
|
|
373
|
+
return HTMLResponse(_base_html("Cost Report", body))
|
|
374
|
+
|
|
375
|
+
from agent_telemetry.models import CostReport as CR
|
|
376
|
+
breakdown = estimate_cost(
|
|
377
|
+
sum(s.input_tokens for s in spans),
|
|
378
|
+
sum(s.output_tokens for s in spans),
|
|
379
|
+
spans[0].model if spans else "unknown",
|
|
380
|
+
sum(s.cache_read for s in spans),
|
|
381
|
+
sum(s.cache_creation for s in spans),
|
|
382
|
+
)
|
|
383
|
+
report = CR(
|
|
384
|
+
session_id=session_id or "all",
|
|
385
|
+
input_cost=breakdown.input_cost,
|
|
386
|
+
output_cost=breakdown.output_cost,
|
|
387
|
+
cache_cost=breakdown.cache_cost,
|
|
388
|
+
total_cost=breakdown.total_cost,
|
|
389
|
+
model=breakdown.model,
|
|
390
|
+
input_tokens=breakdown.input_tokens,
|
|
391
|
+
output_tokens=breakdown.output_tokens,
|
|
392
|
+
cache_read_tokens=breakdown.cache_read_tokens,
|
|
393
|
+
cache_creation_tokens=breakdown.cache_creation_tokens,
|
|
394
|
+
span_count=len(spans),
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
chart = _make_cost_chart(spans)
|
|
398
|
+
|
|
399
|
+
body = f"""
|
|
400
|
+
<h1>Cost Report — {html.escape(report.session_id)}</h1>
|
|
401
|
+
<div class="card">
|
|
402
|
+
<div class="metric"><div class="label">Input Cost</div><div class="value">${report.input_cost:.6f}</div></div>
|
|
403
|
+
<div class="metric"><div class="label">Output Cost</div><div class="value">${report.output_cost:.6f}</div></div>
|
|
404
|
+
<div class="metric"><div class="label">Cache Cost</div><div class="value">${report.cache_cost:.6f}</div></div>
|
|
405
|
+
<div class="metric"><div class="label">Total Cost</div><div class="value">${report.total_cost:.6f}</div></div>
|
|
406
|
+
<div class="metric"><div class="label">Model</div><div class="value">{html.escape(report.model)}</div></div>
|
|
407
|
+
<div class="metric"><div class="label">Spans</div><div class="value">{report.span_count}</div></div>
|
|
408
|
+
</div>
|
|
409
|
+
<div class="card">{chart}</div>
|
|
410
|
+
"""
|
|
411
|
+
return HTMLResponse(_base_html(f"Cost Report — {report.session_id}", body))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
@app.get("/errors/report", response_class=HTMLResponse)
|
|
415
|
+
async def error_report(request: Request, session_id: Optional[str] = None):
|
|
416
|
+
storage = _init_storage()
|
|
417
|
+
sessions = [session_id] if session_id else storage.list_sessions()
|
|
418
|
+
|
|
419
|
+
if not sessions:
|
|
420
|
+
body = '<div class="card"><p>No error data available.</p></div>'
|
|
421
|
+
return HTMLResponse(_base_html("Error Report", body))
|
|
422
|
+
|
|
423
|
+
all_errors_section = ""
|
|
424
|
+
for sid in sessions:
|
|
425
|
+
report = generate_error_report(sid, spans=_load_spans(sid) or None)
|
|
426
|
+
if report.total_errors == 0:
|
|
427
|
+
continue
|
|
428
|
+
|
|
429
|
+
error_rows = ""
|
|
430
|
+
for e in report.errors[:50]:
|
|
431
|
+
badge = "✓" if e.recovered else "✗"
|
|
432
|
+
error_rows += f"""
|
|
433
|
+
<tr>
|
|
434
|
+
<td>{html.escape(e.error_type)}</td>
|
|
435
|
+
<td>{html.escape(e.tool_name)}</td>
|
|
436
|
+
<td>{html.escape(e.error_message[:100])}</td>
|
|
437
|
+
<td>{badge}</td>
|
|
438
|
+
</tr>"""
|
|
439
|
+
|
|
440
|
+
chart = _make_error_chart(sid)
|
|
441
|
+
|
|
442
|
+
all_errors_section += f"""
|
|
443
|
+
<div class="card">
|
|
444
|
+
<h3><a href="/sessions/{ sid}">{html.escape(sid)}</a></h3>
|
|
445
|
+
<div class="metric"><div class="label">Total Errors</div><div class="value">{report.total_errors}</div></div>
|
|
446
|
+
<div class="metric"><div class="label">Recovered</div><div class="value">{report.recovered_errors}</div></div>
|
|
447
|
+
<div class="metric"><div class="label">Recovery Rate</div><div class="value">{report.recovery_rate:.0%}</div></div>
|
|
448
|
+
<table>
|
|
449
|
+
<tr><th>Type</th><th>Tool</th><th>Message</th><th>Recovered</th></tr>
|
|
450
|
+
{error_rows}
|
|
451
|
+
</table>
|
|
452
|
+
{chart}
|
|
453
|
+
</div>"""
|
|
454
|
+
|
|
455
|
+
if not all_errors_section:
|
|
456
|
+
all_errors_section = '<div class="card"><p>No errors found across sessions. 🎉</p></div>'
|
|
457
|
+
|
|
458
|
+
body = f"""
|
|
459
|
+
<h1>Error Report</h1>
|
|
460
|
+
{all_errors_section}
|
|
461
|
+
"""
|
|
462
|
+
return HTMLResponse(_base_html("Error Report", body))
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Error tracking and classification for agent traces."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Optional
|
|
10
|
+
|
|
11
|
+
from agent_telemetry.models import ErrorResponse, ErrorReport, Span, SpanStatus
|
|
12
|
+
from agent_telemetry.collector import ingest_trace
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
ERROR_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|
16
|
+
("bash_error", re.compile(r"(command|bash|shell|exec).*failed|exit code \d+", re.IGNORECASE)),
|
|
17
|
+
("edit_error", re.compile(r"(edit|replace|insert|delete).*fail|not found|no match", re.IGNORECASE)),
|
|
18
|
+
("timeout", re.compile(r"timeout|timed out|deadline exceeded", re.IGNORECASE)),
|
|
19
|
+
("rate_limit", re.compile(r"rate.?limit|429|too many requests", re.IGNORECASE)),
|
|
20
|
+
("auth_error", re.compile(r"auth|401|403|permission denied|forbidden|unauthorized", re.IGNORECASE)),
|
|
21
|
+
("context_overflow", re.compile(r"context.*overflow|context.*length|token.?limit|maximum.*context", re.IGNORECASE)),
|
|
22
|
+
("tool_error", re.compile(r"tool.*(error|fail)|no such tool|invalid.*tool", re.IGNORECASE)),
|
|
23
|
+
("parse_error", re.compile(r"parse|syntax|json.*decode|invalid.*format", re.IGNORECASE)),
|
|
24
|
+
("network_error", re.compile(r"network|connection|dns|socket|ECONNREFUSED|ECONNRESET", re.IGNORECASE)),
|
|
25
|
+
("file_error", re.compile(r"file.*not found|ENOENT|no such file|cannot read", re.IGNORECASE)),
|
|
26
|
+
("memory_error", re.compile(r"out of memory|OOM|memory.*exceeded|heap", re.IGNORECASE)),
|
|
27
|
+
("validation_error", re.compile(r"validation|invalid.*argument|type.*error|value.*error", re.IGNORECASE)),
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def classify_error(error_text: str) -> str:
|
|
32
|
+
"""Classify an error message into a known category.
|
|
33
|
+
|
|
34
|
+
Returns the first matching category or ``"unknown"`` if no pattern matches.
|
|
35
|
+
"""
|
|
36
|
+
if not error_text:
|
|
37
|
+
return "unknown"
|
|
38
|
+
|
|
39
|
+
for category, pattern in ERROR_PATTERNS:
|
|
40
|
+
if pattern.search(error_text):
|
|
41
|
+
return category
|
|
42
|
+
|
|
43
|
+
return "unknown"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def detect_errors(trace_data: list[dict[str, Any]] | list[Span]) -> list[ErrorResponse]:
|
|
47
|
+
"""Find error patterns in a list of spans or raw dicts.
|
|
48
|
+
|
|
49
|
+
For each span with ``status == "error"`` or a non-empty ``error`` field,
|
|
50
|
+
create an ``ErrorResponse`` with the classified error type.
|
|
51
|
+
"""
|
|
52
|
+
errors: list[ErrorResponse] = []
|
|
53
|
+
|
|
54
|
+
for item in trace_data:
|
|
55
|
+
if isinstance(item, Span):
|
|
56
|
+
error_text = item.error or ""
|
|
57
|
+
status = item.status
|
|
58
|
+
tool_name = item.tool_name
|
|
59
|
+
span_id = item.span_id
|
|
60
|
+
session_id = item.session_id
|
|
61
|
+
timestamp = item.timestamp
|
|
62
|
+
elif isinstance(item, dict):
|
|
63
|
+
error_text = item.get("error", "") or ""
|
|
64
|
+
status_val = item.get("status", "ok")
|
|
65
|
+
status = SpanStatus.ERROR if status_val in ("error", "ERROR") else SpanStatus(status_val) if isinstance(status_val, str) and status_val in SpanStatus._value2member_map_ else SpanStatus.OK
|
|
66
|
+
tool_name = item.get("tool_name", "unknown")
|
|
67
|
+
span_id = item.get("span_id", "")
|
|
68
|
+
session_id = item.get("session_id", "unknown")
|
|
69
|
+
ts = item.get("timestamp")
|
|
70
|
+
timestamp = None
|
|
71
|
+
if isinstance(ts, str):
|
|
72
|
+
try:
|
|
73
|
+
timestamp = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
74
|
+
except (ValueError, AttributeError):
|
|
75
|
+
pass
|
|
76
|
+
else:
|
|
77
|
+
continue
|
|
78
|
+
|
|
79
|
+
if status == SpanStatus.ERROR or error_text:
|
|
80
|
+
error_type = classify_error(error_text) if error_text else "unknown"
|
|
81
|
+
errors.append(ErrorResponse(
|
|
82
|
+
span_id=span_id,
|
|
83
|
+
session_id=session_id,
|
|
84
|
+
tool_name=tool_name,
|
|
85
|
+
error_type=error_type,
|
|
86
|
+
error_message=error_text[:500],
|
|
87
|
+
recovered=False,
|
|
88
|
+
timestamp=timestamp,
|
|
89
|
+
))
|
|
90
|
+
|
|
91
|
+
return errors
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def calculate_recovery_rate(errors: list[ErrorResponse], traces: list[Span]) -> float:
|
|
95
|
+
"""Calculate what fraction of errors were followed by a successful operation.
|
|
96
|
+
|
|
97
|
+
An error is considered "recovered" if the next span in the same session
|
|
98
|
+
using the same tool succeeds.
|
|
99
|
+
"""
|
|
100
|
+
if not errors:
|
|
101
|
+
return 1.0
|
|
102
|
+
|
|
103
|
+
session_spans: dict[str, list[Span]] = {}
|
|
104
|
+
for s in traces:
|
|
105
|
+
session_spans.setdefault(s.session_id, []).append(s)
|
|
106
|
+
|
|
107
|
+
for sid in session_spans:
|
|
108
|
+
session_spans[sid].sort(key=lambda s: s.timestamp or datetime.min)
|
|
109
|
+
|
|
110
|
+
recovered = 0
|
|
111
|
+
for err in errors:
|
|
112
|
+
spans = session_spans.get(err.session_id, [])
|
|
113
|
+
for i, span in enumerate(spans):
|
|
114
|
+
if span.span_id == err.span_id and i + 1 < len(spans):
|
|
115
|
+
next_span = spans[i + 1]
|
|
116
|
+
if next_span.tool_name == err.tool_name and next_span.status == SpanStatus.OK:
|
|
117
|
+
err.recovered = True
|
|
118
|
+
recovered += 1
|
|
119
|
+
break
|
|
120
|
+
|
|
121
|
+
return recovered / len(errors)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def generate_error_report(session_id: str, trace_path: Optional[str | Path] = None, spans: Optional[list[Span]] = None) -> ErrorReport:
|
|
125
|
+
"""Generate a full error report for a session.
|
|
126
|
+
|
|
127
|
+
Provide either *trace_path* (to load spans from a file) or *spans* directly.
|
|
128
|
+
"""
|
|
129
|
+
if spans is None and trace_path is not None:
|
|
130
|
+
spans = ingest_trace(trace_path)
|
|
131
|
+
elif spans is None:
|
|
132
|
+
spans = []
|
|
133
|
+
|
|
134
|
+
session_spans = [s for s in spans if s.session_id == session_id]
|
|
135
|
+
errors = detect_errors(session_spans)
|
|
136
|
+
recovery_rate = calculate_recovery_rate(errors, session_spans)
|
|
137
|
+
|
|
138
|
+
by_type: Counter[str] = Counter()
|
|
139
|
+
for e in errors:
|
|
140
|
+
by_type[e.error_type] += 1
|
|
141
|
+
|
|
142
|
+
return ErrorReport(
|
|
143
|
+
session_id=session_id,
|
|
144
|
+
total_errors=len(errors),
|
|
145
|
+
recovered_errors=sum(1 for e in errors if e.recovered),
|
|
146
|
+
recovery_rate=recovery_rate,
|
|
147
|
+
errors_by_type=dict(by_type),
|
|
148
|
+
errors=errors,
|
|
149
|
+
)
|