toad-eye 2.4.5 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -1
- package/dist/agent.js +6 -2
- package/dist/cli.js +8 -1
- package/dist/context/error.d.ts +15 -0
- package/dist/context/error.js +17 -0
- package/dist/context/guard.d.ts +8 -0
- package/dist/context/guard.js +41 -0
- package/dist/context/index.d.ts +1 -0
- package/dist/context/index.js +4 -0
- package/dist/core/metrics.d.ts +4 -1
- package/dist/core/metrics.js +74 -2
- package/dist/core/pricing.d.ts +2 -0
- package/dist/core/pricing.js +61 -16
- package/dist/core/spans.d.ts +2 -0
- package/dist/core/spans.js +48 -9
- package/dist/core/tracer.js +7 -1
- package/dist/instrumentations/anthropic.js +8 -2
- package/dist/instrumentations/create.js +67 -8
- package/dist/instrumentations/gemini.js +7 -0
- package/dist/instrumentations/openai.js +5 -0
- package/dist/instrumentations/types.d.ts +8 -0
- package/dist/mcp/context.d.ts +12 -0
- package/dist/mcp/context.js +22 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +1 -0
- package/dist/mcp/metrics.d.ts +11 -0
- package/dist/mcp/metrics.js +58 -0
- package/dist/mcp/middleware.d.ts +48 -0
- package/dist/mcp/middleware.js +253 -0
- package/dist/mcp/spans.d.ts +28 -0
- package/dist/mcp/spans.js +67 -0
- package/dist/mcp/types.d.ts +34 -0
- package/dist/mcp/types.js +1 -0
- package/dist/types/attributes.d.ts +1 -0
- package/dist/types/attributes.js +1 -0
- package/dist/types/config.d.ts +9 -0
- package/dist/types/metrics.d.ts +8 -0
- package/dist/types/metrics.js +10 -0
- package/dist/types/spans.d.ts +4 -0
- package/package.json +11 -2
- package/templates/docker-compose.yml +4 -4
- package/templates/grafana/dashboards/agent-workflow.json +154 -31
- package/templates/grafana/dashboards/cost-breakdown.json +168 -40
- package/templates/grafana/dashboards/error-drilldown.json +155 -37
- package/templates/grafana/dashboards/finops-attribution.json +192 -43
- package/templates/grafana/dashboards/latency-analysis.json +103 -24
- package/templates/grafana/dashboards/llm-overview.json +268 -66
- package/templates/grafana/dashboards/mcp-server.json +519 -0
- package/templates/grafana/dashboards/model-comparison.json +119 -26
- package/templates/grafana/dashboards/provider-health.json +137 -31
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
{
|
|
2
|
-
"annotations": {
|
|
2
|
+
"annotations": {
|
|
3
|
+
"list": []
|
|
4
|
+
},
|
|
3
5
|
"editable": true,
|
|
4
6
|
"fiscalYearStartMonth": 0,
|
|
5
7
|
"graphTooltip": 1,
|
|
@@ -9,11 +11,19 @@
|
|
|
9
11
|
{
|
|
10
12
|
"title": "Total Requests",
|
|
11
13
|
"type": "stat",
|
|
12
|
-
"gridPos": {
|
|
13
|
-
|
|
14
|
+
"gridPos": {
|
|
15
|
+
"h": 4,
|
|
16
|
+
"w": 6,
|
|
17
|
+
"x": 0,
|
|
18
|
+
"y": 0
|
|
19
|
+
},
|
|
20
|
+
"datasource": {
|
|
21
|
+
"type": "prometheus",
|
|
22
|
+
"uid": "prometheus"
|
|
23
|
+
},
|
|
14
24
|
"targets": [
|
|
15
25
|
{
|
|
16
|
-
"expr": "sum(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
26
|
+
"expr": "sum(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
17
27
|
"refId": "A"
|
|
18
28
|
}
|
|
19
29
|
],
|
|
@@ -22,9 +32,18 @@
|
|
|
22
32
|
"unit": "short",
|
|
23
33
|
"thresholds": {
|
|
24
34
|
"steps": [
|
|
25
|
-
{
|
|
26
|
-
|
|
27
|
-
|
|
35
|
+
{
|
|
36
|
+
"color": "green",
|
|
37
|
+
"value": null
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"color": "yellow",
|
|
41
|
+
"value": 1000
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"color": "red",
|
|
45
|
+
"value": 10000
|
|
46
|
+
}
|
|
28
47
|
]
|
|
29
48
|
}
|
|
30
49
|
},
|
|
@@ -34,11 +53,19 @@
|
|
|
34
53
|
{
|
|
35
54
|
"title": "Error Rate %",
|
|
36
55
|
"type": "stat",
|
|
37
|
-
"gridPos": {
|
|
38
|
-
|
|
56
|
+
"gridPos": {
|
|
57
|
+
"h": 4,
|
|
58
|
+
"w": 6,
|
|
59
|
+
"x": 6,
|
|
60
|
+
"y": 0
|
|
61
|
+
},
|
|
62
|
+
"datasource": {
|
|
63
|
+
"type": "prometheus",
|
|
64
|
+
"uid": "prometheus"
|
|
65
|
+
},
|
|
39
66
|
"targets": [
|
|
40
67
|
{
|
|
41
|
-
"expr": "sum(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}) / sum(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}) * 100",
|
|
68
|
+
"expr": "sum(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}) / sum(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}) * 100",
|
|
42
69
|
"refId": "A"
|
|
43
70
|
}
|
|
44
71
|
],
|
|
@@ -47,9 +74,18 @@
|
|
|
47
74
|
"unit": "percent",
|
|
48
75
|
"thresholds": {
|
|
49
76
|
"steps": [
|
|
50
|
-
{
|
|
51
|
-
|
|
52
|
-
|
|
77
|
+
{
|
|
78
|
+
"color": "green",
|
|
79
|
+
"value": null
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"color": "yellow",
|
|
83
|
+
"value": 5
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"color": "red",
|
|
87
|
+
"value": 20
|
|
88
|
+
}
|
|
53
89
|
]
|
|
54
90
|
}
|
|
55
91
|
},
|
|
@@ -59,11 +95,19 @@
|
|
|
59
95
|
{
|
|
60
96
|
"title": "Avg Latency",
|
|
61
97
|
"type": "stat",
|
|
62
|
-
"gridPos": {
|
|
63
|
-
|
|
98
|
+
"gridPos": {
|
|
99
|
+
"h": 4,
|
|
100
|
+
"w": 6,
|
|
101
|
+
"x": 12,
|
|
102
|
+
"y": 0
|
|
103
|
+
},
|
|
104
|
+
"datasource": {
|
|
105
|
+
"type": "prometheus",
|
|
106
|
+
"uid": "prometheus"
|
|
107
|
+
},
|
|
64
108
|
"targets": [
|
|
65
109
|
{
|
|
66
|
-
"expr": "sum(rate(gen_ai_client_operation_duration_milliseconds_sum{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) / sum(rate(gen_ai_client_operation_duration_milliseconds_count{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m]))",
|
|
110
|
+
"expr": "sum(rate(gen_ai_client_operation_duration_milliseconds_sum{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) / sum(rate(gen_ai_client_operation_duration_milliseconds_count{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m]))",
|
|
67
111
|
"refId": "A"
|
|
68
112
|
}
|
|
69
113
|
],
|
|
@@ -72,9 +116,18 @@
|
|
|
72
116
|
"unit": "ms",
|
|
73
117
|
"thresholds": {
|
|
74
118
|
"steps": [
|
|
75
|
-
{
|
|
76
|
-
|
|
77
|
-
|
|
119
|
+
{
|
|
120
|
+
"color": "green",
|
|
121
|
+
"value": null
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
"color": "yellow",
|
|
125
|
+
"value": 1000
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"color": "red",
|
|
129
|
+
"value": 5000
|
|
130
|
+
}
|
|
78
131
|
]
|
|
79
132
|
}
|
|
80
133
|
},
|
|
@@ -84,11 +137,19 @@
|
|
|
84
137
|
{
|
|
85
138
|
"title": "Total Cost",
|
|
86
139
|
"type": "stat",
|
|
87
|
-
"gridPos": {
|
|
88
|
-
|
|
140
|
+
"gridPos": {
|
|
141
|
+
"h": 4,
|
|
142
|
+
"w": 6,
|
|
143
|
+
"x": 18,
|
|
144
|
+
"y": 0
|
|
145
|
+
},
|
|
146
|
+
"datasource": {
|
|
147
|
+
"type": "prometheus",
|
|
148
|
+
"uid": "prometheus"
|
|
149
|
+
},
|
|
89
150
|
"targets": [
|
|
90
151
|
{
|
|
91
|
-
"expr": "sum(gen_ai_client_request_cost_USD_sum{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
152
|
+
"expr": "sum(gen_ai_client_request_cost_USD_sum{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
92
153
|
"refId": "A"
|
|
93
154
|
}
|
|
94
155
|
],
|
|
@@ -97,9 +158,18 @@
|
|
|
97
158
|
"unit": "currencyUSD",
|
|
98
159
|
"thresholds": {
|
|
99
160
|
"steps": [
|
|
100
|
-
{
|
|
101
|
-
|
|
102
|
-
|
|
161
|
+
{
|
|
162
|
+
"color": "green",
|
|
163
|
+
"value": null
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"color": "yellow",
|
|
167
|
+
"value": 10
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
"color": "red",
|
|
171
|
+
"value": 100
|
|
172
|
+
}
|
|
103
173
|
]
|
|
104
174
|
}
|
|
105
175
|
},
|
|
@@ -109,11 +179,19 @@
|
|
|
109
179
|
{
|
|
110
180
|
"title": "Request Rate per minute",
|
|
111
181
|
"type": "timeseries",
|
|
112
|
-
"gridPos": {
|
|
113
|
-
|
|
182
|
+
"gridPos": {
|
|
183
|
+
"h": 8,
|
|
184
|
+
"w": 12,
|
|
185
|
+
"x": 0,
|
|
186
|
+
"y": 4
|
|
187
|
+
},
|
|
188
|
+
"datasource": {
|
|
189
|
+
"type": "prometheus",
|
|
190
|
+
"uid": "prometheus"
|
|
191
|
+
},
|
|
114
192
|
"targets": [
|
|
115
193
|
{
|
|
116
|
-
"expr": "sum(rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
194
|
+
"expr": "sum(rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
117
195
|
"legendFormat": "{{ gen_ai_provider_name }}",
|
|
118
196
|
"refId": "A"
|
|
119
197
|
}
|
|
@@ -121,7 +199,11 @@
|
|
|
121
199
|
"fieldConfig": {
|
|
122
200
|
"defaults": {
|
|
123
201
|
"unit": "reqpm",
|
|
124
|
-
"custom": {
|
|
202
|
+
"custom": {
|
|
203
|
+
"drawStyle": "line",
|
|
204
|
+
"fillOpacity": 15,
|
|
205
|
+
"lineWidth": 2
|
|
206
|
+
}
|
|
125
207
|
},
|
|
126
208
|
"overrides": []
|
|
127
209
|
}
|
|
@@ -129,11 +211,19 @@
|
|
|
129
211
|
{
|
|
130
212
|
"title": "Error Rate per minute",
|
|
131
213
|
"type": "timeseries",
|
|
132
|
-
"gridPos": {
|
|
133
|
-
|
|
214
|
+
"gridPos": {
|
|
215
|
+
"h": 8,
|
|
216
|
+
"w": 12,
|
|
217
|
+
"x": 12,
|
|
218
|
+
"y": 4
|
|
219
|
+
},
|
|
220
|
+
"datasource": {
|
|
221
|
+
"type": "prometheus",
|
|
222
|
+
"uid": "prometheus"
|
|
223
|
+
},
|
|
134
224
|
"targets": [
|
|
135
225
|
{
|
|
136
|
-
"expr": "sum(rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
226
|
+
"expr": "sum(rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
137
227
|
"legendFormat": "{{ gen_ai_provider_name }}",
|
|
138
228
|
"refId": "A"
|
|
139
229
|
}
|
|
@@ -141,8 +231,14 @@
|
|
|
141
231
|
"fieldConfig": {
|
|
142
232
|
"defaults": {
|
|
143
233
|
"unit": "reqpm",
|
|
144
|
-
"custom": {
|
|
145
|
-
|
|
234
|
+
"custom": {
|
|
235
|
+
"drawStyle": "line",
|
|
236
|
+
"fillOpacity": 15,
|
|
237
|
+
"lineWidth": 2
|
|
238
|
+
},
|
|
239
|
+
"color": {
|
|
240
|
+
"mode": "palette-classic"
|
|
241
|
+
}
|
|
146
242
|
},
|
|
147
243
|
"overrides": []
|
|
148
244
|
}
|
|
@@ -150,16 +246,24 @@
|
|
|
150
246
|
{
|
|
151
247
|
"title": "Latency p50 / p95 (ms)",
|
|
152
248
|
"type": "timeseries",
|
|
153
|
-
"gridPos": {
|
|
154
|
-
|
|
249
|
+
"gridPos": {
|
|
250
|
+
"h": 8,
|
|
251
|
+
"w": 12,
|
|
252
|
+
"x": 0,
|
|
253
|
+
"y": 12
|
|
254
|
+
},
|
|
255
|
+
"datasource": {
|
|
256
|
+
"type": "prometheus",
|
|
257
|
+
"uid": "prometheus"
|
|
258
|
+
},
|
|
155
259
|
"targets": [
|
|
156
260
|
{
|
|
157
|
-
"expr": "histogram_quantile(0.50, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le))",
|
|
261
|
+
"expr": "histogram_quantile(0.50, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le))",
|
|
158
262
|
"legendFormat": "p50",
|
|
159
263
|
"refId": "A"
|
|
160
264
|
},
|
|
161
265
|
{
|
|
162
|
-
"expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le))",
|
|
266
|
+
"expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le))",
|
|
163
267
|
"legendFormat": "p95",
|
|
164
268
|
"refId": "B"
|
|
165
269
|
}
|
|
@@ -167,7 +271,11 @@
|
|
|
167
271
|
"fieldConfig": {
|
|
168
272
|
"defaults": {
|
|
169
273
|
"unit": "ms",
|
|
170
|
-
"custom": {
|
|
274
|
+
"custom": {
|
|
275
|
+
"drawStyle": "line",
|
|
276
|
+
"fillOpacity": 15,
|
|
277
|
+
"lineWidth": 2
|
|
278
|
+
}
|
|
171
279
|
},
|
|
172
280
|
"overrides": []
|
|
173
281
|
}
|
|
@@ -175,11 +283,19 @@
|
|
|
175
283
|
{
|
|
176
284
|
"title": "Cost per minute (USD)",
|
|
177
285
|
"type": "timeseries",
|
|
178
|
-
"gridPos": {
|
|
179
|
-
|
|
286
|
+
"gridPos": {
|
|
287
|
+
"h": 8,
|
|
288
|
+
"w": 12,
|
|
289
|
+
"x": 12,
|
|
290
|
+
"y": 12
|
|
291
|
+
},
|
|
292
|
+
"datasource": {
|
|
293
|
+
"type": "prometheus",
|
|
294
|
+
"uid": "prometheus"
|
|
295
|
+
},
|
|
180
296
|
"targets": [
|
|
181
297
|
{
|
|
182
|
-
"expr": "sum(rate(gen_ai_client_request_cost_USD_sum{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
298
|
+
"expr": "sum(rate(gen_ai_client_request_cost_USD_sum{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_provider_name) * 60",
|
|
183
299
|
"legendFormat": "{{ gen_ai_provider_name }}",
|
|
184
300
|
"refId": "A"
|
|
185
301
|
}
|
|
@@ -187,7 +303,11 @@
|
|
|
187
303
|
"fieldConfig": {
|
|
188
304
|
"defaults": {
|
|
189
305
|
"unit": "currencyUSD",
|
|
190
|
-
"custom": {
|
|
306
|
+
"custom": {
|
|
307
|
+
"drawStyle": "bars",
|
|
308
|
+
"fillOpacity": 50,
|
|
309
|
+
"lineWidth": 1
|
|
310
|
+
}
|
|
191
311
|
},
|
|
192
312
|
"overrides": []
|
|
193
313
|
}
|
|
@@ -195,11 +315,19 @@
|
|
|
195
315
|
{
|
|
196
316
|
"title": "Total Tokens",
|
|
197
317
|
"type": "stat",
|
|
198
|
-
"gridPos": {
|
|
199
|
-
|
|
318
|
+
"gridPos": {
|
|
319
|
+
"h": 4,
|
|
320
|
+
"w": 8,
|
|
321
|
+
"x": 0,
|
|
322
|
+
"y": 20
|
|
323
|
+
},
|
|
324
|
+
"datasource": {
|
|
325
|
+
"type": "prometheus",
|
|
326
|
+
"uid": "prometheus"
|
|
327
|
+
},
|
|
200
328
|
"targets": [
|
|
201
329
|
{
|
|
202
|
-
"expr": "sum(gen_ai_client_token_usage_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
330
|
+
"expr": "sum(gen_ai_client_token_usage_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
203
331
|
"refId": "A"
|
|
204
332
|
}
|
|
205
333
|
],
|
|
@@ -208,9 +336,18 @@
|
|
|
208
336
|
"unit": "short",
|
|
209
337
|
"thresholds": {
|
|
210
338
|
"steps": [
|
|
211
|
-
{
|
|
212
|
-
|
|
213
|
-
|
|
339
|
+
{
|
|
340
|
+
"color": "green",
|
|
341
|
+
"value": null
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
"color": "yellow",
|
|
345
|
+
"value": 100000
|
|
346
|
+
},
|
|
347
|
+
{
|
|
348
|
+
"color": "red",
|
|
349
|
+
"value": 1000000
|
|
350
|
+
}
|
|
214
351
|
]
|
|
215
352
|
}
|
|
216
353
|
},
|
|
@@ -220,11 +357,19 @@
|
|
|
220
357
|
{
|
|
221
358
|
"title": "Total Requests",
|
|
222
359
|
"type": "stat",
|
|
223
|
-
"gridPos": {
|
|
224
|
-
|
|
360
|
+
"gridPos": {
|
|
361
|
+
"h": 4,
|
|
362
|
+
"w": 8,
|
|
363
|
+
"x": 8,
|
|
364
|
+
"y": 20
|
|
365
|
+
},
|
|
366
|
+
"datasource": {
|
|
367
|
+
"type": "prometheus",
|
|
368
|
+
"uid": "prometheus"
|
|
369
|
+
},
|
|
225
370
|
"targets": [
|
|
226
371
|
{
|
|
227
|
-
"expr": "sum(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
372
|
+
"expr": "sum(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
228
373
|
"refId": "A"
|
|
229
374
|
}
|
|
230
375
|
],
|
|
@@ -233,9 +378,18 @@
|
|
|
233
378
|
"unit": "short",
|
|
234
379
|
"thresholds": {
|
|
235
380
|
"steps": [
|
|
236
|
-
{
|
|
237
|
-
|
|
238
|
-
|
|
381
|
+
{
|
|
382
|
+
"color": "green",
|
|
383
|
+
"value": null
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
"color": "yellow",
|
|
387
|
+
"value": 1000
|
|
388
|
+
},
|
|
389
|
+
{
|
|
390
|
+
"color": "red",
|
|
391
|
+
"value": 10000
|
|
392
|
+
}
|
|
239
393
|
]
|
|
240
394
|
}
|
|
241
395
|
},
|
|
@@ -245,11 +399,19 @@
|
|
|
245
399
|
{
|
|
246
400
|
"title": "Total Errors",
|
|
247
401
|
"type": "stat",
|
|
248
|
-
"gridPos": {
|
|
249
|
-
|
|
402
|
+
"gridPos": {
|
|
403
|
+
"h": 4,
|
|
404
|
+
"w": 8,
|
|
405
|
+
"x": 16,
|
|
406
|
+
"y": 20
|
|
407
|
+
},
|
|
408
|
+
"datasource": {
|
|
409
|
+
"type": "prometheus",
|
|
410
|
+
"uid": "prometheus"
|
|
411
|
+
},
|
|
250
412
|
"targets": [
|
|
251
413
|
{
|
|
252
|
-
"expr": "sum(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
414
|
+
"expr": "sum(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"})",
|
|
253
415
|
"refId": "A"
|
|
254
416
|
}
|
|
255
417
|
],
|
|
@@ -258,9 +420,18 @@
|
|
|
258
420
|
"unit": "short",
|
|
259
421
|
"thresholds": {
|
|
260
422
|
"steps": [
|
|
261
|
-
{
|
|
262
|
-
|
|
263
|
-
|
|
423
|
+
{
|
|
424
|
+
"color": "green",
|
|
425
|
+
"value": null
|
|
426
|
+
},
|
|
427
|
+
{
|
|
428
|
+
"color": "orange",
|
|
429
|
+
"value": 10
|
|
430
|
+
},
|
|
431
|
+
{
|
|
432
|
+
"color": "red",
|
|
433
|
+
"value": 100
|
|
434
|
+
}
|
|
264
435
|
]
|
|
265
436
|
}
|
|
266
437
|
},
|
|
@@ -273,29 +444,60 @@
|
|
|
273
444
|
"tags": ["llm", "observability", "toad-eye"],
|
|
274
445
|
"templating": {
|
|
275
446
|
"list": [
|
|
447
|
+
{
|
|
448
|
+
"name": "service",
|
|
449
|
+
"type": "query",
|
|
450
|
+
"datasource": {
|
|
451
|
+
"type": "prometheus",
|
|
452
|
+
"uid": "prometheus"
|
|
453
|
+
},
|
|
454
|
+
"query": "label_values(gen_ai_client_requests_total, job)",
|
|
455
|
+
"includeAll": true,
|
|
456
|
+
"allValue": ".*",
|
|
457
|
+
"current": {
|
|
458
|
+
"text": "All",
|
|
459
|
+
"value": "$__all"
|
|
460
|
+
},
|
|
461
|
+
"multi": true
|
|
462
|
+
},
|
|
276
463
|
{
|
|
277
464
|
"name": "provider",
|
|
278
465
|
"type": "query",
|
|
279
|
-
"datasource": {
|
|
466
|
+
"datasource": {
|
|
467
|
+
"type": "prometheus",
|
|
468
|
+
"uid": "prometheus"
|
|
469
|
+
},
|
|
280
470
|
"query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
|
|
281
471
|
"includeAll": true,
|
|
282
472
|
"allValue": ".*",
|
|
283
|
-
"current": {
|
|
473
|
+
"current": {
|
|
474
|
+
"text": "All",
|
|
475
|
+
"value": "$__all"
|
|
476
|
+
},
|
|
284
477
|
"multi": true
|
|
285
478
|
},
|
|
286
479
|
{
|
|
287
480
|
"name": "model",
|
|
288
481
|
"type": "query",
|
|
289
|
-
"datasource": {
|
|
482
|
+
"datasource": {
|
|
483
|
+
"type": "prometheus",
|
|
484
|
+
"uid": "prometheus"
|
|
485
|
+
},
|
|
290
486
|
"query": "label_values(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}, gen_ai_request_model)",
|
|
291
487
|
"includeAll": true,
|
|
292
488
|
"allValue": ".*",
|
|
293
|
-
"current": {
|
|
489
|
+
"current": {
|
|
490
|
+
"text": "All",
|
|
491
|
+
"value": "$__all"
|
|
492
|
+
},
|
|
294
493
|
"multi": true
|
|
295
494
|
}
|
|
296
495
|
]
|
|
297
496
|
},
|
|
298
|
-
"time": {
|
|
497
|
+
"time": {
|
|
498
|
+
"from": "now-15m",
|
|
499
|
+
"to": "now"
|
|
500
|
+
},
|
|
299
501
|
"timepicker": {},
|
|
300
502
|
"timezone": "browser",
|
|
301
503
|
"title": "toad-eye: Overview",
|