toad-eye 2.10.0 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alerts/conditions.js +1 -0
- package/dist/cli.js +76 -8
- package/dist/core/console-exporter.d.ts +17 -0
- package/dist/core/console-exporter.js +55 -0
- package/dist/core/metrics.d.ts +1 -0
- package/dist/core/metrics.js +12 -0
- package/dist/core/spans.d.ts +4 -0
- package/dist/core/spans.js +21 -1
- package/dist/core/tracer.js +27 -16
- package/dist/instrumentations/create.js +0 -1
- package/dist/mcp/client.js +6 -0
- package/dist/mcp/index.d.ts +1 -0
- package/dist/mcp/index.js +1 -0
- package/dist/mcp/metrics.d.ts +1 -0
- package/dist/mcp/metrics.js +11 -0
- package/dist/types/config.d.ts +2 -0
- package/dist/types/metrics.d.ts +2 -0
- package/dist/types/metrics.js +3 -0
- package/package.json +1 -1
- package/templates/docker-compose.yml +4 -4
- package/templates/grafana/dashboards/agent-workflow.json +25 -8
- package/templates/grafana/dashboards/cost-breakdown.json +14 -7
- package/templates/grafana/dashboards/error-drilldown.json +12 -6
- package/templates/grafana/dashboards/finops-attribution.json +38 -8
- package/templates/grafana/dashboards/latency-analysis.json +8 -4
- package/templates/grafana/dashboards/llm-overview.json +22 -11
- package/templates/grafana/dashboards/mcp-analytics.json +176 -43
- package/templates/grafana/dashboards/mcp-e2e.json +195 -49
- package/templates/grafana/dashboards/mcp-server.json +46 -9
- package/templates/grafana/dashboards/model-comparison.json +10 -5
- package/templates/grafana/dashboards/provider-health.json +10 -5
|
@@ -12,8 +12,16 @@
|
|
|
12
12
|
"title": "Tool Popularity Ranking",
|
|
13
13
|
"description": "Most called tools by total invocations",
|
|
14
14
|
"type": "bargauge",
|
|
15
|
-
"gridPos": {
|
|
16
|
-
|
|
15
|
+
"gridPos": {
|
|
16
|
+
"h": 8,
|
|
17
|
+
"w": 12,
|
|
18
|
+
"x": 0,
|
|
19
|
+
"y": 0
|
|
20
|
+
},
|
|
21
|
+
"datasource": {
|
|
22
|
+
"type": "prometheus",
|
|
23
|
+
"uid": "prometheus"
|
|
24
|
+
},
|
|
17
25
|
"targets": [
|
|
18
26
|
{
|
|
19
27
|
"expr": "topk(10, sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_calls_total[1h])))",
|
|
@@ -28,12 +36,25 @@
|
|
|
28
36
|
"decimals": 0,
|
|
29
37
|
"thresholds": {
|
|
30
38
|
"steps": [
|
|
31
|
-
{
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
39
|
+
{
|
|
40
|
+
"color": "blue",
|
|
41
|
+
"value": null
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"color": "green",
|
|
45
|
+
"value": 10
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"color": "yellow",
|
|
49
|
+
"value": 100
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"color": "red",
|
|
53
|
+
"value": 500
|
|
54
|
+
}
|
|
35
55
|
]
|
|
36
|
-
}
|
|
56
|
+
},
|
|
57
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
37
58
|
},
|
|
38
59
|
"overrides": []
|
|
39
60
|
},
|
|
@@ -49,8 +70,16 @@
|
|
|
49
70
|
"title": "Unique Callers per Tool",
|
|
50
71
|
"description": "Number of distinct sessions calling each tool",
|
|
51
72
|
"type": "bargauge",
|
|
52
|
-
"gridPos": {
|
|
53
|
-
|
|
73
|
+
"gridPos": {
|
|
74
|
+
"h": 8,
|
|
75
|
+
"w": 12,
|
|
76
|
+
"x": 12,
|
|
77
|
+
"y": 0
|
|
78
|
+
},
|
|
79
|
+
"datasource": {
|
|
80
|
+
"type": "prometheus",
|
|
81
|
+
"uid": "prometheus"
|
|
82
|
+
},
|
|
54
83
|
"targets": [
|
|
55
84
|
{
|
|
56
85
|
"expr": "count by (gen_ai_tool_name) (sum by (gen_ai_tool_name, mcp_session_id) (gen_ai_mcp_tool_callers_total))",
|
|
@@ -65,11 +94,21 @@
|
|
|
65
94
|
"decimals": 0,
|
|
66
95
|
"thresholds": {
|
|
67
96
|
"steps": [
|
|
68
|
-
{
|
|
69
|
-
|
|
70
|
-
|
|
97
|
+
{
|
|
98
|
+
"color": "purple",
|
|
99
|
+
"value": null
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"color": "blue",
|
|
103
|
+
"value": 5
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"color": "green",
|
|
107
|
+
"value": 20
|
|
108
|
+
}
|
|
71
109
|
]
|
|
72
|
-
}
|
|
110
|
+
},
|
|
111
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
73
112
|
},
|
|
74
113
|
"overrides": []
|
|
75
114
|
},
|
|
@@ -85,8 +124,16 @@
|
|
|
85
124
|
"title": "Tool Usage Over Time",
|
|
86
125
|
"description": "Call rate per tool — spot trends and spikes",
|
|
87
126
|
"type": "timeseries",
|
|
88
|
-
"gridPos": {
|
|
89
|
-
|
|
127
|
+
"gridPos": {
|
|
128
|
+
"h": 8,
|
|
129
|
+
"w": 24,
|
|
130
|
+
"x": 0,
|
|
131
|
+
"y": 8
|
|
132
|
+
},
|
|
133
|
+
"datasource": {
|
|
134
|
+
"type": "prometheus",
|
|
135
|
+
"uid": "prometheus"
|
|
136
|
+
},
|
|
90
137
|
"targets": [
|
|
91
138
|
{
|
|
92
139
|
"expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_calls_total[5m]))",
|
|
@@ -103,9 +150,14 @@
|
|
|
103
150
|
"fillOpacity": 20,
|
|
104
151
|
"pointSize": 5,
|
|
105
152
|
"showPoints": "auto",
|
|
106
|
-
"stacking": {
|
|
153
|
+
"stacking": {
|
|
154
|
+
"mode": "none"
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
"color": {
|
|
158
|
+
"mode": "palette-classic"
|
|
107
159
|
},
|
|
108
|
-
"
|
|
160
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
109
161
|
},
|
|
110
162
|
"overrides": []
|
|
111
163
|
},
|
|
@@ -115,15 +167,25 @@
|
|
|
115
167
|
"placement": "right",
|
|
116
168
|
"calcs": ["mean", "max", "sum"]
|
|
117
169
|
},
|
|
118
|
-
"tooltip": {
|
|
170
|
+
"tooltip": {
|
|
171
|
+
"mode": "multi"
|
|
172
|
+
}
|
|
119
173
|
}
|
|
120
174
|
},
|
|
121
175
|
{
|
|
122
176
|
"title": "Cost per LLM Model (via sampling)",
|
|
123
177
|
"description": "LLM cost breakdown — visible when MCP tools call LLMs internally",
|
|
124
178
|
"type": "piechart",
|
|
125
|
-
"gridPos": {
|
|
126
|
-
|
|
179
|
+
"gridPos": {
|
|
180
|
+
"h": 8,
|
|
181
|
+
"w": 8,
|
|
182
|
+
"x": 0,
|
|
183
|
+
"y": 16
|
|
184
|
+
},
|
|
185
|
+
"datasource": {
|
|
186
|
+
"type": "prometheus",
|
|
187
|
+
"uid": "prometheus"
|
|
188
|
+
},
|
|
127
189
|
"targets": [
|
|
128
190
|
{
|
|
129
191
|
"expr": "sum by (gen_ai_request_model) (increase(gen_ai_client_request_cost_USD_sum[1h]))",
|
|
@@ -136,7 +198,10 @@
|
|
|
136
198
|
"defaults": {
|
|
137
199
|
"unit": "currencyUSD",
|
|
138
200
|
"decimals": 4,
|
|
139
|
-
"color": {
|
|
201
|
+
"color": {
|
|
202
|
+
"mode": "palette-classic"
|
|
203
|
+
},
|
|
204
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
140
205
|
},
|
|
141
206
|
"overrides": []
|
|
142
207
|
},
|
|
@@ -147,15 +212,25 @@
|
|
|
147
212
|
"values": ["value", "percent"]
|
|
148
213
|
},
|
|
149
214
|
"pieType": "donut",
|
|
150
|
-
"tooltip": {
|
|
215
|
+
"tooltip": {
|
|
216
|
+
"mode": "multi"
|
|
217
|
+
}
|
|
151
218
|
}
|
|
152
219
|
},
|
|
153
220
|
{
|
|
154
221
|
"title": "Cost Over Time",
|
|
155
222
|
"description": "LLM cost rate — attribute to tools via trace context",
|
|
156
223
|
"type": "timeseries",
|
|
157
|
-
"gridPos": {
|
|
158
|
-
|
|
224
|
+
"gridPos": {
|
|
225
|
+
"h": 8,
|
|
226
|
+
"w": 16,
|
|
227
|
+
"x": 8,
|
|
228
|
+
"y": 16
|
|
229
|
+
},
|
|
230
|
+
"datasource": {
|
|
231
|
+
"type": "prometheus",
|
|
232
|
+
"uid": "prometheus"
|
|
233
|
+
},
|
|
159
234
|
"targets": [
|
|
160
235
|
{
|
|
161
236
|
"expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_request_cost_USD_sum[5m])) * 3600",
|
|
@@ -174,7 +249,10 @@
|
|
|
174
249
|
"pointSize": 5,
|
|
175
250
|
"showPoints": "auto"
|
|
176
251
|
},
|
|
177
|
-
"color": {
|
|
252
|
+
"color": {
|
|
253
|
+
"mode": "palette-classic"
|
|
254
|
+
},
|
|
255
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
178
256
|
},
|
|
179
257
|
"overrides": []
|
|
180
258
|
},
|
|
@@ -184,15 +262,25 @@
|
|
|
184
262
|
"placement": "bottom",
|
|
185
263
|
"calcs": ["sum", "mean"]
|
|
186
264
|
},
|
|
187
|
-
"tooltip": {
|
|
265
|
+
"tooltip": {
|
|
266
|
+
"mode": "multi"
|
|
267
|
+
}
|
|
188
268
|
}
|
|
189
269
|
},
|
|
190
270
|
{
|
|
191
271
|
"title": "Tool Performance Summary",
|
|
192
272
|
"description": "All tools: call count, avg duration, error rate, unique callers",
|
|
193
273
|
"type": "table",
|
|
194
|
-
"gridPos": {
|
|
195
|
-
|
|
274
|
+
"gridPos": {
|
|
275
|
+
"h": 8,
|
|
276
|
+
"w": 24,
|
|
277
|
+
"x": 0,
|
|
278
|
+
"y": 24
|
|
279
|
+
},
|
|
280
|
+
"datasource": {
|
|
281
|
+
"type": "prometheus",
|
|
282
|
+
"uid": "prometheus"
|
|
283
|
+
},
|
|
196
284
|
"targets": [
|
|
197
285
|
{
|
|
198
286
|
"expr": "sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_calls_total[1h]))",
|
|
@@ -224,11 +312,15 @@
|
|
|
224
312
|
}
|
|
225
313
|
],
|
|
226
314
|
"transformations": [
|
|
227
|
-
{
|
|
315
|
+
{
|
|
316
|
+
"id": "merge"
|
|
317
|
+
},
|
|
228
318
|
{
|
|
229
319
|
"id": "organize",
|
|
230
320
|
"options": {
|
|
231
|
-
"excludeByName": {
|
|
321
|
+
"excludeByName": {
|
|
322
|
+
"Time": true
|
|
323
|
+
},
|
|
232
324
|
"renameByName": {
|
|
233
325
|
"gen_ai_tool_name": "Tool",
|
|
234
326
|
"Value #calls": "Calls (1h)",
|
|
@@ -241,38 +333,79 @@
|
|
|
241
333
|
],
|
|
242
334
|
"fieldConfig": {
|
|
243
335
|
"defaults": {
|
|
244
|
-
"custom": {
|
|
336
|
+
"custom": {
|
|
337
|
+
"align": "center"
|
|
338
|
+
},
|
|
339
|
+
"noValue": "No data yet — run: npx toad-eye demo"
|
|
245
340
|
},
|
|
246
341
|
"overrides": [
|
|
247
342
|
{
|
|
248
|
-
"matcher": {
|
|
343
|
+
"matcher": {
|
|
344
|
+
"id": "byName",
|
|
345
|
+
"options": "Avg Duration (ms)"
|
|
346
|
+
},
|
|
249
347
|
"properties": [
|
|
250
|
-
{
|
|
251
|
-
|
|
348
|
+
{
|
|
349
|
+
"id": "unit",
|
|
350
|
+
"value": "ms"
|
|
351
|
+
},
|
|
352
|
+
{
|
|
353
|
+
"id": "decimals",
|
|
354
|
+
"value": 1
|
|
355
|
+
}
|
|
252
356
|
]
|
|
253
357
|
},
|
|
254
358
|
{
|
|
255
|
-
"matcher": {
|
|
359
|
+
"matcher": {
|
|
360
|
+
"id": "byName",
|
|
361
|
+
"options": "Error Rate"
|
|
362
|
+
},
|
|
256
363
|
"properties": [
|
|
257
|
-
{
|
|
258
|
-
|
|
364
|
+
{
|
|
365
|
+
"id": "unit",
|
|
366
|
+
"value": "percentunit"
|
|
367
|
+
},
|
|
368
|
+
{
|
|
369
|
+
"id": "decimals",
|
|
370
|
+
"value": 2
|
|
371
|
+
},
|
|
259
372
|
{
|
|
260
373
|
"id": "thresholds",
|
|
261
374
|
"value": {
|
|
262
375
|
"mode": "absolute",
|
|
263
376
|
"steps": [
|
|
264
|
-
{
|
|
265
|
-
|
|
266
|
-
|
|
377
|
+
{
|
|
378
|
+
"color": "green",
|
|
379
|
+
"value": null
|
|
380
|
+
},
|
|
381
|
+
{
|
|
382
|
+
"color": "yellow",
|
|
383
|
+
"value": 0.05
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
"color": "red",
|
|
387
|
+
"value": 0.1
|
|
388
|
+
}
|
|
267
389
|
]
|
|
268
390
|
}
|
|
269
391
|
},
|
|
270
|
-
{
|
|
392
|
+
{
|
|
393
|
+
"id": "custom.displayMode",
|
|
394
|
+
"value": "color-background"
|
|
395
|
+
}
|
|
271
396
|
]
|
|
272
397
|
},
|
|
273
398
|
{
|
|
274
|
-
"matcher": {
|
|
275
|
-
|
|
399
|
+
"matcher": {
|
|
400
|
+
"id": "byName",
|
|
401
|
+
"options": "Calls (1h)"
|
|
402
|
+
},
|
|
403
|
+
"properties": [
|
|
404
|
+
{
|
|
405
|
+
"id": "decimals",
|
|
406
|
+
"value": 0
|
|
407
|
+
}
|
|
408
|
+
]
|
|
276
409
|
}
|
|
277
410
|
]
|
|
278
411
|
}
|