toad-eye 2.5.0 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +33 -0
  2. package/dist/cli.js +8 -1
  3. package/dist/context/index.d.ts +1 -2
  4. package/dist/context/index.js +4 -2
  5. package/dist/core/metrics.js +42 -0
  6. package/dist/core/pricing.js +10 -1
  7. package/dist/core/spans.d.ts +2 -0
  8. package/dist/core/spans.js +24 -10
  9. package/dist/core/tracer.js +7 -3
  10. package/dist/index.d.ts +0 -1
  11. package/dist/index.js +0 -1
  12. package/dist/instrumentations/create.js +23 -8
  13. package/dist/instrumentations/gemini.js +7 -0
  14. package/dist/instrumentations/openai.js +5 -0
  15. package/dist/instrumentations/types.d.ts +6 -0
  16. package/dist/mcp/context.d.ts +12 -0
  17. package/dist/mcp/context.js +22 -0
  18. package/dist/mcp/index.d.ts +3 -0
  19. package/dist/mcp/index.js +1 -0
  20. package/dist/mcp/metrics.d.ts +11 -0
  21. package/dist/mcp/metrics.js +58 -0
  22. package/dist/mcp/middleware.d.ts +48 -0
  23. package/dist/mcp/middleware.js +253 -0
  24. package/dist/mcp/spans.d.ts +28 -0
  25. package/dist/mcp/spans.js +67 -0
  26. package/dist/mcp/types.d.ts +34 -0
  27. package/dist/mcp/types.js +1 -0
  28. package/dist/types/config.d.ts +5 -3
  29. package/dist/types/metrics.d.ts +5 -0
  30. package/dist/types/metrics.js +6 -0
  31. package/package.json +11 -2
  32. package/templates/docker-compose.yml +4 -4
  33. package/templates/grafana/dashboards/mcp-server.json +519 -0
package/templates/grafana/dashboards/mcp-server.json ADDED
@@ -0,0 +1,519 @@
1
+ {
2
+ "annotations": {
3
+ "list": []
4
+ },
5
+ "editable": true,
6
+ "fiscalYearStartMonth": 0,
7
+ "graphTooltip": 1,
8
+ "id": null,
9
+ "links": [],
10
+ "panels": [
11
+ {
12
+ "title": "Tool Call Rate",
13
+ "type": "stat",
14
+ "gridPos": {
15
+ "h": 4,
16
+ "w": 6,
17
+ "x": 0,
18
+ "y": 0
19
+ },
20
+ "datasource": {
21
+ "type": "prometheus",
22
+ "uid": "prometheus"
23
+ },
24
+ "targets": [
25
+ {
26
+ "expr": "sum(rate(gen_ai_mcp_tool_calls_total[5m]))",
27
+ "refId": "A"
28
+ }
29
+ ],
30
+ "fieldConfig": {
31
+ "defaults": {
32
+ "unit": "reqps",
33
+ "decimals": 2,
34
+ "thresholds": {
35
+ "steps": [
36
+ {
37
+ "color": "green",
38
+ "value": null
39
+ },
40
+ {
41
+ "color": "yellow",
42
+ "value": 50
43
+ },
44
+ {
45
+ "color": "red",
46
+ "value": 100
47
+ }
48
+ ]
49
+ }
50
+ },
51
+ "overrides": []
52
+ }
53
+ },
54
+ {
55
+ "title": "Avg Tool Duration",
56
+ "type": "stat",
57
+ "gridPos": {
58
+ "h": 4,
59
+ "w": 6,
60
+ "x": 6,
61
+ "y": 0
62
+ },
63
+ "datasource": {
64
+ "type": "prometheus",
65
+ "uid": "prometheus"
66
+ },
67
+ "targets": [
68
+ {
69
+ "expr": "sum(rate(gen_ai_mcp_tool_duration_sum[5m])) / sum(rate(gen_ai_mcp_tool_duration_count[5m])) or vector(0)",
70
+ "refId": "A"
71
+ }
72
+ ],
73
+ "fieldConfig": {
74
+ "defaults": {
75
+ "unit": "ms",
76
+ "decimals": 1,
77
+ "thresholds": {
78
+ "steps": [
79
+ {
80
+ "color": "green",
81
+ "value": null
82
+ },
83
+ {
84
+ "color": "yellow",
85
+ "value": 500
86
+ },
87
+ {
88
+ "color": "red",
89
+ "value": 2000
90
+ }
91
+ ]
92
+ }
93
+ },
94
+ "overrides": []
95
+ }
96
+ },
97
+ {
98
+ "title": "Error Rate",
99
+ "type": "stat",
100
+ "gridPos": {
101
+ "h": 4,
102
+ "w": 6,
103
+ "x": 12,
104
+ "y": 0
105
+ },
106
+ "datasource": {
107
+ "type": "prometheus",
108
+ "uid": "prometheus"
109
+ },
110
+ "targets": [
111
+ {
112
+ "expr": "sum(rate(gen_ai_mcp_tool_errors_total[5m])) / sum(rate(gen_ai_mcp_tool_calls_total[5m])) or vector(0)",
113
+ "refId": "A"
114
+ }
115
+ ],
116
+ "fieldConfig": {
117
+ "defaults": {
118
+ "unit": "percentunit",
119
+ "decimals": 2,
120
+ "thresholds": {
121
+ "steps": [
122
+ {
123
+ "color": "green",
124
+ "value": null
125
+ },
126
+ {
127
+ "color": "yellow",
128
+ "value": 0.05
129
+ },
130
+ {
131
+ "color": "red",
132
+ "value": 0.1
133
+ }
134
+ ]
135
+ }
136
+ },
137
+ "overrides": []
138
+ }
139
+ },
140
+ {
141
+ "title": "Resource Reads",
142
+ "type": "stat",
143
+ "gridPos": {
144
+ "h": 4,
145
+ "w": 6,
146
+ "x": 18,
147
+ "y": 0
148
+ },
149
+ "datasource": {
150
+ "type": "prometheus",
151
+ "uid": "prometheus"
152
+ },
153
+ "targets": [
154
+ {
155
+ "expr": "sum(rate(gen_ai_mcp_resource_reads_total[5m]))",
156
+ "refId": "A"
157
+ }
158
+ ],
159
+ "fieldConfig": {
160
+ "defaults": {
161
+ "unit": "reqps",
162
+ "decimals": 2,
163
+ "thresholds": {
164
+ "steps": [
165
+ {
166
+ "color": "blue",
167
+ "value": null
168
+ }
169
+ ]
170
+ }
171
+ },
172
+ "overrides": []
173
+ }
174
+ },
175
+ {
176
+ "title": "Tool Call Rate by Tool",
177
+ "type": "timeseries",
178
+ "gridPos": {
179
+ "h": 8,
180
+ "w": 12,
181
+ "x": 0,
182
+ "y": 4
183
+ },
184
+ "datasource": {
185
+ "type": "prometheus",
186
+ "uid": "prometheus"
187
+ },
188
+ "targets": [
189
+ {
190
+ "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_calls_total[5m]))",
191
+ "legendFormat": "{{gen_ai_tool_name}}",
192
+ "refId": "A"
193
+ }
194
+ ],
195
+ "fieldConfig": {
196
+ "defaults": {
197
+ "unit": "reqps",
198
+ "custom": {
199
+ "drawStyle": "line",
200
+ "lineInterpolation": "smooth",
201
+ "fillOpacity": 15,
202
+ "pointSize": 5,
203
+ "showPoints": "auto"
204
+ }
205
+ },
206
+ "overrides": []
207
+ },
208
+ "options": {
209
+ "legend": {
210
+ "displayMode": "table",
211
+ "placement": "bottom",
212
+ "calcs": ["mean", "max"]
213
+ },
214
+ "tooltip": {
215
+ "mode": "multi"
216
+ }
217
+ }
218
+ },
219
+ {
220
+ "title": "Tool Duration p50 / p95",
221
+ "type": "timeseries",
222
+ "gridPos": {
223
+ "h": 8,
224
+ "w": 12,
225
+ "x": 12,
226
+ "y": 4
227
+ },
228
+ "datasource": {
229
+ "type": "prometheus",
230
+ "uid": "prometheus"
231
+ },
232
+ "targets": [
233
+ {
234
+ "expr": "histogram_quantile(0.50, sum by (gen_ai_tool_name, le) (rate(gen_ai_mcp_tool_duration_bucket[5m])))",
235
+ "legendFormat": "p50 {{gen_ai_tool_name}}",
236
+ "refId": "A"
237
+ },
238
+ {
239
+ "expr": "histogram_quantile(0.95, sum by (gen_ai_tool_name, le) (rate(gen_ai_mcp_tool_duration_bucket[5m])))",
240
+ "legendFormat": "p95 {{gen_ai_tool_name}}",
241
+ "refId": "B"
242
+ }
243
+ ],
244
+ "fieldConfig": {
245
+ "defaults": {
246
+ "unit": "ms",
247
+ "custom": {
248
+ "drawStyle": "line",
249
+ "lineInterpolation": "smooth",
250
+ "fillOpacity": 10,
251
+ "pointSize": 5,
252
+ "showPoints": "auto"
253
+ }
254
+ },
255
+ "overrides": []
256
+ },
257
+ "options": {
258
+ "legend": {
259
+ "displayMode": "table",
260
+ "placement": "bottom",
261
+ "calcs": ["mean", "max"]
262
+ },
263
+ "tooltip": {
264
+ "mode": "multi"
265
+ }
266
+ }
267
+ },
268
+ {
269
+ "title": "Errors by Tool",
270
+ "type": "timeseries",
271
+ "gridPos": {
272
+ "h": 8,
273
+ "w": 12,
274
+ "x": 0,
275
+ "y": 12
276
+ },
277
+ "datasource": {
278
+ "type": "prometheus",
279
+ "uid": "prometheus"
280
+ },
281
+ "targets": [
282
+ {
283
+ "expr": "sum by (gen_ai_tool_name, error_type) (rate(gen_ai_mcp_tool_errors_total[5m]))",
284
+ "legendFormat": "{{gen_ai_tool_name}} ({{error_type}})",
285
+ "refId": "A"
286
+ }
287
+ ],
288
+ "fieldConfig": {
289
+ "defaults": {
290
+ "unit": "reqps",
291
+ "custom": {
292
+ "drawStyle": "bars",
293
+ "fillOpacity": 50,
294
+ "stacking": {
295
+ "mode": "normal"
296
+ }
297
+ },
298
+ "color": {
299
+ "mode": "palette-classic"
300
+ }
301
+ },
302
+ "overrides": []
303
+ },
304
+ "options": {
305
+ "legend": {
306
+ "displayMode": "table",
307
+ "placement": "bottom",
308
+ "calcs": ["sum"]
309
+ },
310
+ "tooltip": {
311
+ "mode": "multi"
312
+ }
313
+ }
314
+ },
315
+ {
316
+ "title": "Resource Reads by URI",
317
+ "type": "timeseries",
318
+ "gridPos": {
319
+ "h": 8,
320
+ "w": 12,
321
+ "x": 12,
322
+ "y": 12
323
+ },
324
+ "datasource": {
325
+ "type": "prometheus",
326
+ "uid": "prometheus"
327
+ },
328
+ "targets": [
329
+ {
330
+ "expr": "sum by (gen_ai_data_source_id) (rate(gen_ai_mcp_resource_reads_total[5m]))",
331
+ "legendFormat": "{{gen_ai_data_source_id}}",
332
+ "refId": "A"
333
+ }
334
+ ],
335
+ "fieldConfig": {
336
+ "defaults": {
337
+ "unit": "reqps",
338
+ "custom": {
339
+ "drawStyle": "line",
340
+ "lineInterpolation": "smooth",
341
+ "fillOpacity": 15,
342
+ "pointSize": 5,
343
+ "showPoints": "auto"
344
+ }
345
+ },
346
+ "overrides": []
347
+ },
348
+ "options": {
349
+ "legend": {
350
+ "displayMode": "table",
351
+ "placement": "bottom",
352
+ "calcs": ["mean", "sum"]
353
+ },
354
+ "tooltip": {
355
+ "mode": "multi"
356
+ }
357
+ }
358
+ },
359
+ {
360
+ "title": "Tool Performance Overview",
361
+ "type": "table",
362
+ "gridPos": {
363
+ "h": 8,
364
+ "w": 24,
365
+ "x": 0,
366
+ "y": 20
367
+ },
368
+ "datasource": {
369
+ "type": "prometheus",
370
+ "uid": "prometheus"
371
+ },
372
+ "targets": [
373
+ {
374
+ "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_calls_total[5m]))",
375
+ "legendFormat": "",
376
+ "refId": "calls",
377
+ "format": "table",
378
+ "instant": true
379
+ },
380
+ {
381
+ "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_sum[5m])) / sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_count[5m]))",
382
+ "legendFormat": "",
383
+ "refId": "avg_duration",
384
+ "format": "table",
385
+ "instant": true
386
+ },
387
+ {
388
+ "expr": "histogram_quantile(0.95, sum by (gen_ai_tool_name, le) (rate(gen_ai_mcp_tool_duration_bucket[5m])))",
389
+ "legendFormat": "",
390
+ "refId": "p95_duration",
391
+ "format": "table",
392
+ "instant": true
393
+ },
394
+ {
395
+ "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_errors_total[5m])) / sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_calls_total[5m]))",
396
+ "legendFormat": "",
397
+ "refId": "error_rate",
398
+ "format": "table",
399
+ "instant": true
400
+ }
401
+ ],
402
+ "transformations": [
403
+ {
404
+ "id": "merge"
405
+ },
406
+ {
407
+ "id": "organize",
408
+ "options": {
409
+ "excludeByName": {
410
+ "Time": true
411
+ },
412
+ "renameByName": {
413
+ "gen_ai_tool_name": "Tool",
414
+ "Value #calls": "Call Rate (req/s)",
415
+ "Value #avg_duration": "Avg Duration (ms)",
416
+ "Value #p95_duration": "p95 Duration (ms)",
417
+ "Value #error_rate": "Error Rate"
418
+ }
419
+ }
420
+ }
421
+ ],
422
+ "fieldConfig": {
423
+ "defaults": {
424
+ "custom": {
425
+ "align": "center"
426
+ }
427
+ },
428
+ "overrides": [
429
+ {
430
+ "matcher": {
431
+ "id": "byName",
432
+ "options": "Avg Duration (ms)"
433
+ },
434
+ "properties": [
435
+ {
436
+ "id": "unit",
437
+ "value": "ms"
438
+ },
439
+ {
440
+ "id": "decimals",
441
+ "value": 1
442
+ }
443
+ ]
444
+ },
445
+ {
446
+ "matcher": {
447
+ "id": "byName",
448
+ "options": "p95 Duration (ms)"
449
+ },
450
+ "properties": [
451
+ {
452
+ "id": "unit",
453
+ "value": "ms"
454
+ },
455
+ {
456
+ "id": "decimals",
457
+ "value": 1
458
+ }
459
+ ]
460
+ },
461
+ {
462
+ "matcher": {
463
+ "id": "byName",
464
+ "options": "Error Rate"
465
+ },
466
+ "properties": [
467
+ {
468
+ "id": "unit",
469
+ "value": "percentunit"
470
+ },
471
+ {
472
+ "id": "decimals",
473
+ "value": 2
474
+ },
475
+ {
476
+ "id": "thresholds",
477
+ "value": {
478
+ "mode": "absolute",
479
+ "steps": [
480
+ {
481
+ "color": "green",
482
+ "value": null
483
+ },
484
+ {
485
+ "color": "yellow",
486
+ "value": 0.05
487
+ },
488
+ {
489
+ "color": "red",
490
+ "value": 0.1
491
+ }
492
+ ]
493
+ }
494
+ },
495
+ {
496
+ "id": "custom.displayMode",
497
+ "value": "color-background"
498
+ }
499
+ ]
500
+ }
501
+ ]
502
+ }
503
+ }
504
+ ],
505
+ "schemaVersion": 39,
506
+ "tags": ["toad-eye", "mcp", "llm"],
507
+ "templating": {
508
+ "list": []
509
+ },
510
+ "time": {
511
+ "from": "now-1h",
512
+ "to": "now"
513
+ },
514
+ "timepicker": {},
515
+ "timezone": "browser",
516
+ "title": "MCP Server",
517
+ "uid": "toad-eye-mcp-server",
518
+ "version": 1
519
+ }