fadenstack 0.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. faden_cli/__init__.py +16 -0
  2. faden_cli/__main__.py +9 -0
  3. faden_cli/bundle/shared/.gitignore +25 -0
  4. faden_cli/bundle/shared/LICENSE +187 -0
  5. faden_cli/bundle/shared/NOTICE +15 -0
  6. faden_cli/bundle/shared/OFFICIAL.md +9 -0
  7. faden_cli/bundle/shared/README.md +16 -0
  8. faden_cli/bundle/shared/agent-binaries/.gitignore +5 -0
  9. faden_cli/bundle/shared/alloy/config.alloy +40 -0
  10. faden_cli/bundle/shared/clickhouse/config.d/system-logs.xml +26 -0
  11. faden_cli/bundle/shared/docker-compose.gpu.yaml +34 -0
  12. faden_cli/bundle/shared/docker-compose.yaml +998 -0
  13. faden_cli/bundle/shared/grafana/dashboards/templates/vllm-runtime.json +2099 -0
  14. faden_cli/bundle/shared/grafana/provisioning/dashboards/dashboard.yml +10 -0
  15. faden_cli/bundle/shared/grafana/provisioning/dashboards/json/faden-overview.json +211 -0
  16. faden_cli/bundle/shared/grafana/provisioning/datasources/ds.yaml +9 -0
  17. faden_cli/bundle/shared/grafana/provisioning/datasources/prometheus.yaml +15 -0
  18. faden_cli/bundle/shared/initdb/01-init.sql +32 -0
  19. faden_cli/bundle/shared/loki/local-config.yaml +32 -0
  20. faden_cli/bundle/shared/nginx/locations.conf +245 -0
  21. faden_cli/bundle/shared/nginx/maintenance.html +239 -0
  22. faden_cli/bundle/shared/nginx/nginx-tls.conf +79 -0
  23. faden_cli/bundle/shared/nginx/nginx.conf +44 -0
  24. faden_cli/bundle/shared/nginx/rate-limits.conf +42 -0
  25. faden_cli/bundle/shared/nginx/security-headers.conf +22 -0
  26. faden_cli/bundle/shared/nginx/select-config.sh +37 -0
  27. faden_cli/bundle/shared/prometheus/prometheus.yml +40 -0
  28. faden_cli/bundle/shared/rabbitmq/definitions.json.example +37 -0
  29. faden_cli/bundle/shared/tls/README.md +30 -0
  30. faden_cli/cli.py +158 -0
  31. faden_cli/commands/__init__.py +1 -0
  32. faden_cli/commands/admin.py +161 -0
  33. faden_cli/commands/backup.py +159 -0
  34. faden_cli/commands/config.py +146 -0
  35. faden_cli/commands/deploy.py +949 -0
  36. faden_cli/commands/dev/__init__.py +1 -0
  37. faden_cli/commands/dev/dev_doctor.py +46 -0
  38. faden_cli/commands/dev/dev_down.py +241 -0
  39. faden_cli/commands/dev/dev_group.py +24 -0
  40. faden_cli/commands/dev/dev_init.py +710 -0
  41. faden_cli/commands/dev/dev_pull.py +297 -0
  42. faden_cli/commands/dev/dev_status.py +151 -0
  43. faden_cli/commands/dev/dev_up.py +1392 -0
  44. faden_cli/commands/doctor.py +225 -0
  45. faden_cli/commands/down.py +75 -0
  46. faden_cli/commands/logs_cmd.py +41 -0
  47. faden_cli/commands/maintenance.py +74 -0
  48. faden_cli/commands/module.py +113 -0
  49. faden_cli/commands/observe.py +95 -0
  50. faden_cli/commands/ports.py +91 -0
  51. faden_cli/commands/rebuild.py +158 -0
  52. faden_cli/commands/restore.py +139 -0
  53. faden_cli/commands/runtime_images.py +174 -0
  54. faden_cli/commands/status.py +71 -0
  55. faden_cli/commands/tls.py +722 -0
  56. faden_cli/commands/tune.py +37 -0
  57. faden_cli/commands/up.py +64 -0
  58. faden_cli/commands/upgrade.py +470 -0
  59. faden_cli/commands/version.py +78 -0
  60. faden_cli/core/__init__.py +1 -0
  61. faden_cli/core/api_client.py +77 -0
  62. faden_cli/core/backup.py +717 -0
  63. faden_cli/core/bootstrap.py +291 -0
  64. faden_cli/core/bundle.py +110 -0
  65. faden_cli/core/clickhouse.py +61 -0
  66. faden_cli/core/compose.py +484 -0
  67. faden_cli/core/console.py +106 -0
  68. faden_cli/core/detect.py +580 -0
  69. faden_cli/core/dev_modules.py +149 -0
  70. faden_cli/core/docker_env.py +147 -0
  71. faden_cli/core/edge.py +94 -0
  72. faden_cli/core/env_gen.py +598 -0
  73. faden_cli/core/git.py +219 -0
  74. faden_cli/core/install.py +288 -0
  75. faden_cli/core/installation.py +186 -0
  76. faden_cli/core/local_node.py +719 -0
  77. faden_cli/core/maintenance.py +82 -0
  78. faden_cli/core/private_files.py +133 -0
  79. faden_cli/core/registry.py +574 -0
  80. faden_cli/core/release_feed.py +342 -0
  81. faden_cli/core/rmq.py +135 -0
  82. faden_cli/core/runtime_images.py +309 -0
  83. faden_cli/core/schedule.py +278 -0
  84. faden_cli/core/settings.py +159 -0
  85. faden_cli/core/sysinfo.py +303 -0
  86. faden_cli/core/tls.py +790 -0
  87. faden_cli/core/tune.py +154 -0
  88. faden_cli/core/update_check.py +254 -0
  89. faden_cli/core/workspace.py +155 -0
  90. faden_cli/templates/env.j2 +33 -0
  91. faden_cli/tui/__init__.py +1 -0
  92. faden_cli/tui/widgets/__init__.py +1 -0
  93. faden_cli/tui/wizard/__init__.py +1 -0
  94. fadenstack-0.4.2.dist-info/METADATA +304 -0
  95. fadenstack-0.4.2.dist-info/RECORD +99 -0
  96. fadenstack-0.4.2.dist-info/WHEEL +4 -0
  97. fadenstack-0.4.2.dist-info/entry_points.txt +3 -0
  98. fadenstack-0.4.2.dist-info/licenses/LICENSE +187 -0
  99. fadenstack-0.4.2.dist-info/licenses/NOTICE +15 -0
faden_cli/bundle/shared/grafana/dashboards/templates/vllm-runtime.json ADDED
@@ -0,0 +1,2099 @@
1
+ {
2
+ "annotations": {
3
+ "list": [
4
+ {
5
+ "builtIn": 1,
6
+ "datasource": {
7
+ "type": "grafana",
8
+ "uid": "-- Grafana --"
9
+ },
10
+ "enable": true,
11
+ "hide": true,
12
+ "iconColor": "rgba(0, 211, 255, 1)",
13
+ "name": "Annotations & Alerts",
14
+ "type": "dashboard"
15
+ }
16
+ ]
17
+ },
18
+ "description": "vLLM performance dashboard tailored for DGX Spark TP2 + MTP3 + prefix caching. Metrics match the current vLLM /metrics endpoint.",
19
+ "editable": true,
20
+ "fiscalYearStartMonth": 0,
21
+ "graphTooltip": 1,
22
+ "id": null,
23
+ "links": [],
24
+ "liveNow": false,
25
+ "panels": [
26
+ {
27
+ "id": 0,
28
+ "type": "text",
29
+ "title": "",
30
+ "gridPos": {
31
+ "h": 2,
32
+ "w": 24,
33
+ "x": 0,
34
+ "y": 0
35
+ },
36
+ "options": {
37
+ "code": {
38
+ "language": "plaintext",
39
+ "showLineNumbers": false,
40
+ "showMiniMap": false
41
+ },
42
+ "content": "Auto-provisioned by fadenstack for **__RUNTIME_NAME__**. This is generated from a template, so edits made here are lost the next time it is provisioned — change `shared/grafana/dashboards/templates/vllm-runtime.json` instead. It is removed when its runtime or cluster is.",
43
+ "mode": "markdown"
44
+ }
45
+ },
46
+ {
47
+ "type": "stat",
48
+ "title": "Running requests",
49
+ "datasource": {
50
+ "type": "prometheus",
51
+ "uid": "prometheus"
52
+ },
53
+ "gridPos": {
54
+ "x": 0,
55
+ "y": 2,
56
+ "w": 4,
57
+ "h": 4
58
+ },
59
+ "targets": [
60
+ {
61
+ "datasource": {
62
+ "type": "prometheus",
63
+ "uid": "prometheus"
64
+ },
65
+ "expr": "(sum(last_over_time({__name__=~\"vllm:num_requests_running|ray_vllm_num_requests_running\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
66
+ "refId": "A",
67
+ "legendFormat": "",
68
+ "instant": false,
69
+ "range": true
70
+ }
71
+ ],
72
+ "fieldConfig": {
73
+ "defaults": {
74
+ "unit": "short",
75
+ "color": {
76
+ "mode": "thresholds"
77
+ },
78
+ "thresholds": {
79
+ "mode": "absolute",
80
+ "steps": [
81
+ {
82
+ "color": "green",
83
+ "value": null
84
+ }
85
+ ]
86
+ },
87
+ "mappings": []
88
+ },
89
+ "overrides": []
90
+ },
91
+ "options": {
92
+ "reduceOptions": {
93
+ "values": false,
94
+ "calcs": [
95
+ "lastNotNull"
96
+ ],
97
+ "fields": ""
98
+ },
99
+ "orientation": "auto",
100
+ "textMode": "auto",
101
+ "colorMode": "value",
102
+ "graphMode": "area",
103
+ "justifyMode": "auto"
104
+ },
105
+ "description": "Requests currently in model execution batches.",
106
+ "id": 1
107
+ },
108
+ {
109
+ "type": "stat",
110
+ "title": "Waiting requests",
111
+ "datasource": {
112
+ "type": "prometheus",
113
+ "uid": "prometheus"
114
+ },
115
+ "gridPos": {
116
+ "x": 4,
117
+ "y": 2,
118
+ "w": 4,
119
+ "h": 4
120
+ },
121
+ "targets": [
122
+ {
123
+ "datasource": {
124
+ "type": "prometheus",
125
+ "uid": "prometheus"
126
+ },
127
+ "expr": "(sum(last_over_time({__name__=~\"vllm:num_requests_waiting|ray_vllm_num_requests_waiting\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
128
+ "refId": "A",
129
+ "legendFormat": "",
130
+ "instant": false,
131
+ "range": true
132
+ }
133
+ ],
134
+ "fieldConfig": {
135
+ "defaults": {
136
+ "unit": "short",
137
+ "color": {
138
+ "mode": "thresholds"
139
+ },
140
+ "thresholds": {
141
+ "mode": "absolute",
142
+ "steps": [
143
+ {
144
+ "color": "green",
145
+ "value": null
146
+ }
147
+ ]
148
+ },
149
+ "mappings": []
150
+ },
151
+ "overrides": []
152
+ },
153
+ "options": {
154
+ "reduceOptions": {
155
+ "values": false,
156
+ "calcs": [
157
+ "lastNotNull"
158
+ ],
159
+ "fields": ""
160
+ },
161
+ "orientation": "auto",
162
+ "textMode": "auto",
163
+ "colorMode": "value",
164
+ "graphMode": "area",
165
+ "justifyMode": "auto"
166
+ },
167
+ "description": "Requests waiting for scheduling capacity or transient constraints.",
168
+ "id": 2
169
+ },
170
+ {
171
+ "type": "stat",
172
+ "title": "KV cache usage",
173
+ "datasource": {
174
+ "type": "prometheus",
175
+ "uid": "prometheus"
176
+ },
177
+ "gridPos": {
178
+ "x": 8,
179
+ "y": 2,
180
+ "w": 4,
181
+ "h": 4
182
+ },
183
+ "targets": [
184
+ {
185
+ "datasource": {
186
+ "type": "prometheus",
187
+ "uid": "prometheus"
188
+ },
189
+ "expr": "(100 * avg(last_over_time({__name__=~\"vllm:kv_cache_usage_perc|ray_vllm_kv_cache_usage_perc\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
190
+ "refId": "A",
191
+ "legendFormat": "",
192
+ "instant": false,
193
+ "range": true
194
+ }
195
+ ],
196
+ "fieldConfig": {
197
+ "defaults": {
198
+ "unit": "percent",
199
+ "color": {
200
+ "mode": "thresholds"
201
+ },
202
+ "thresholds": {
203
+ "mode": "absolute",
204
+ "steps": [
205
+ {
206
+ "color": "green",
207
+ "value": null
208
+ },
209
+ {
210
+ "color": "yellow",
211
+ "value": 70
212
+ },
213
+ {
214
+ "color": "red",
215
+ "value": 90
216
+ }
217
+ ]
218
+ },
219
+ "mappings": [],
220
+ "decimals": 1
221
+ },
222
+ "overrides": []
223
+ },
224
+ "options": {
225
+ "reduceOptions": {
226
+ "values": false,
227
+ "calcs": [
228
+ "lastNotNull"
229
+ ],
230
+ "fields": ""
231
+ },
232
+ "orientation": "auto",
233
+ "textMode": "auto",
234
+ "colorMode": "value",
235
+ "graphMode": "area",
236
+ "justifyMode": "auto"
237
+ },
238
+ "id": 3
239
+ },
240
+ {
241
+ "type": "stat",
242
+ "title": "Prefix cache hit rate",
243
+ "datasource": {
244
+ "type": "prometheus",
245
+ "uid": "prometheus"
246
+ },
247
+ "gridPos": {
248
+ "x": 12,
249
+ "y": 2,
250
+ "w": 4,
251
+ "h": 4
252
+ },
253
+ "targets": [
254
+ {
255
+ "datasource": {
256
+ "type": "prometheus",
257
+ "uid": "prometheus"
258
+ },
259
+ "expr": "100 * sum(rate({__name__=~\"vllm:prefix_cache_hits_total|ray_vllm_prefix_cache_hits_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:prefix_cache_queries_total|ray_vllm_prefix_cache_queries_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
260
+ "refId": "A",
261
+ "legendFormat": "",
262
+ "instant": false,
263
+ "range": true
264
+ }
265
+ ],
266
+ "fieldConfig": {
267
+ "defaults": {
268
+ "unit": "percent",
269
+ "color": {
270
+ "mode": "thresholds"
271
+ },
272
+ "thresholds": {
273
+ "mode": "absolute",
274
+ "steps": [
275
+ {
276
+ "color": "green",
277
+ "value": null
278
+ }
279
+ ]
280
+ },
281
+ "mappings": [],
282
+ "decimals": 1
283
+ },
284
+ "overrides": []
285
+ },
286
+ "options": {
287
+ "reduceOptions": {
288
+ "values": false,
289
+ "calcs": [
290
+ "lastNotNull"
291
+ ],
292
+ "fields": ""
293
+ },
294
+ "orientation": "auto",
295
+ "textMode": "auto",
296
+ "colorMode": "value",
297
+ "graphMode": "area",
298
+ "justifyMode": "auto"
299
+ },
300
+ "id": 4
301
+ },
302
+ {
303
+ "type": "stat",
304
+ "title": "MTP acceptance",
305
+ "datasource": {
306
+ "type": "prometheus",
307
+ "uid": "prometheus"
308
+ },
309
+ "gridPos": {
310
+ "x": 16,
311
+ "y": 2,
312
+ "w": 4,
313
+ "h": 4
314
+ },
315
+ "targets": [
316
+ {
317
+ "datasource": {
318
+ "type": "prometheus",
319
+ "uid": "prometheus"
320
+ },
321
+ "expr": "100 * sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_total|ray_vllm_spec_decode_num_accepted_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_draft_tokens_total|ray_vllm_spec_decode_num_draft_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
322
+ "refId": "A",
323
+ "legendFormat": "",
324
+ "instant": false,
325
+ "range": true
326
+ }
327
+ ],
328
+ "fieldConfig": {
329
+ "defaults": {
330
+ "unit": "percent",
331
+ "color": {
332
+ "mode": "thresholds"
333
+ },
334
+ "thresholds": {
335
+ "mode": "absolute",
336
+ "steps": [
337
+ {
338
+ "color": "green",
339
+ "value": null
340
+ }
341
+ ]
342
+ },
343
+ "mappings": [],
344
+ "decimals": 1
345
+ },
346
+ "overrides": []
347
+ },
348
+ "options": {
349
+ "reduceOptions": {
350
+ "values": false,
351
+ "calcs": [
352
+ "lastNotNull"
353
+ ],
354
+ "fields": ""
355
+ },
356
+ "orientation": "auto",
357
+ "textMode": "auto",
358
+ "colorMode": "value",
359
+ "graphMode": "area",
360
+ "justifyMode": "auto"
361
+ },
362
+ "description": "Accepted speculative tokens / drafted speculative tokens over 5 minutes.",
363
+ "id": 5
364
+ },
365
+ {
366
+ "type": "stat",
367
+ "title": "Generation tok/s",
368
+ "datasource": {
369
+ "type": "prometheus",
370
+ "uid": "prometheus"
371
+ },
372
+ "gridPos": {
373
+ "x": 20,
374
+ "y": 2,
375
+ "w": 4,
376
+ "h": 4
377
+ },
378
+ "targets": [
379
+ {
380
+ "datasource": {
381
+ "type": "prometheus",
382
+ "uid": "prometheus"
383
+ },
384
+ "expr": "sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[1m]))",
385
+ "refId": "A",
386
+ "legendFormat": "",
387
+ "instant": false,
388
+ "range": true
389
+ }
390
+ ],
391
+ "fieldConfig": {
392
+ "defaults": {
393
+ "unit": "short",
394
+ "color": {
395
+ "mode": "thresholds"
396
+ },
397
+ "thresholds": {
398
+ "mode": "absolute",
399
+ "steps": [
400
+ {
401
+ "color": "green",
402
+ "value": null
403
+ }
404
+ ]
405
+ },
406
+ "mappings": [],
407
+ "decimals": 1
408
+ },
409
+ "overrides": []
410
+ },
411
+ "options": {
412
+ "reduceOptions": {
413
+ "values": false,
414
+ "calcs": [
415
+ "lastNotNull"
416
+ ],
417
+ "fields": ""
418
+ },
419
+ "orientation": "auto",
420
+ "textMode": "auto",
421
+ "colorMode": "value",
422
+ "graphMode": "area",
423
+ "justifyMode": "auto"
424
+ },
425
+ "description": "Aggregate output token throughput over the last minute.",
426
+ "id": 6
427
+ },
428
+ {
429
+ "type": "timeseries",
430
+ "title": "Token throughput",
431
+ "datasource": {
432
+ "type": "prometheus",
433
+ "uid": "prometheus"
434
+ },
435
+ "gridPos": {
436
+ "x": 0,
437
+ "y": 6,
438
+ "w": 12,
439
+ "h": 8
440
+ },
441
+ "targets": [
442
+ {
443
+ "datasource": {
444
+ "type": "prometheus",
445
+ "uid": "prometheus"
446
+ },
447
+ "expr": "sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[1m]))",
448
+ "refId": "A",
449
+ "legendFormat": "generation tok/s"
450
+ },
451
+ {
452
+ "datasource": {
453
+ "type": "prometheus",
454
+ "uid": "prometheus"
455
+ },
456
+ "expr": "(sum(rate({__name__=~\"vllm:prompt_tokens_by_source_total|ray_vllm_prompt_tokens_by_source_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",source=\"local_compute\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
457
+ "refId": "B",
458
+ "legendFormat": "prefill compute tok/s"
459
+ },
460
+ {
461
+ "datasource": {
462
+ "type": "prometheus",
463
+ "uid": "prometheus"
464
+ },
465
+ "expr": "(sum(rate({__name__=~\"vllm:prompt_tokens_by_source_total|ray_vllm_prompt_tokens_by_source_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",source=\"local_cache_hit\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
466
+ "refId": "C",
467
+ "legendFormat": "cached prompt tok/s"
468
+ }
469
+ ],
470
+ "fieldConfig": {
471
+ "defaults": {
472
+ "unit": "short",
473
+ "color": {
474
+ "mode": "palette-classic"
475
+ },
476
+ "custom": {
477
+ "drawStyle": "line",
478
+ "lineInterpolation": "linear",
479
+ "barAlignment": 0,
480
+ "lineWidth": 1,
481
+ "fillOpacity": 8,
482
+ "gradientMode": "none",
483
+ "spanNulls": false,
484
+ "showPoints": "auto",
485
+ "pointSize": 5,
486
+ "stacking": {
487
+ "mode": "none",
488
+ "group": "A"
489
+ },
490
+ "axisPlacement": "auto",
491
+ "axisLabel": "",
492
+ "axisColorMode": "text",
493
+ "scaleDistribution": {
494
+ "type": "linear"
495
+ },
496
+ "hideFrom": {
497
+ "tooltip": false,
498
+ "viz": false,
499
+ "legend": false
500
+ },
501
+ "thresholdsStyle": {
502
+ "mode": "off"
503
+ }
504
+ },
505
+ "thresholds": {
506
+ "mode": "absolute",
507
+ "steps": [
508
+ {
509
+ "color": "green",
510
+ "value": null
511
+ }
512
+ ]
513
+ },
514
+ "mappings": []
515
+ },
516
+ "overrides": []
517
+ },
518
+ "options": {
519
+ "tooltip": {
520
+ "mode": "multi",
521
+ "sort": "desc"
522
+ },
523
+ "legend": {
524
+ "displayMode": "table",
525
+ "placement": "bottom",
526
+ "calcs": [
527
+ "lastNotNull",
528
+ "mean",
529
+ "max"
530
+ ]
531
+ }
532
+ },
533
+ "description": "Output generation throughput plus prompt tokens computed locally and served from prefix cache.",
534
+ "id": 7
535
+ },
536
+ {
537
+ "type": "timeseries",
538
+ "title": "Request & HTTP rate",
539
+ "datasource": {
540
+ "type": "prometheus",
541
+ "uid": "prometheus"
542
+ },
543
+ "gridPos": {
544
+ "x": 12,
545
+ "y": 6,
546
+ "w": 12,
547
+ "h": 8
548
+ },
549
+ "targets": [
550
+ {
551
+ "datasource": {
552
+ "type": "prometheus",
553
+ "uid": "prometheus"
554
+ },
555
+ "expr": "(sum(rate(ray_serve_num_http_requests_total{route=~\".*/v1/chat/completions\",status_code=~\"2..\",runtime_name=\"__RUNTIME_NAME__\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
556
+ "refId": "A",
557
+ "legendFormat": "2xx req/s"
558
+ },
559
+ {
560
+ "datasource": {
561
+ "type": "prometheus",
562
+ "uid": "prometheus"
563
+ },
564
+ "expr": "(sum(rate(ray_serve_num_http_requests_total{route=~\".*/v1/chat/completions\",status_code=~\"4..\",runtime_name=\"__RUNTIME_NAME__\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
565
+ "refId": "B",
566
+ "legendFormat": "4xx req/s"
567
+ },
568
+ {
569
+ "datasource": {
570
+ "type": "prometheus",
571
+ "uid": "prometheus"
572
+ },
573
+ "expr": "(sum(rate(ray_serve_num_http_requests_total{route=~\".*/v1/chat/completions\",status_code=~\"5..\",runtime_name=\"__RUNTIME_NAME__\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
574
+ "refId": "C",
575
+ "legendFormat": "5xx req/s"
576
+ }
577
+ ],
578
+ "fieldConfig": {
579
+ "defaults": {
580
+ "unit": "short",
581
+ "color": {
582
+ "mode": "palette-classic"
583
+ },
584
+ "custom": {
585
+ "drawStyle": "line",
586
+ "lineInterpolation": "linear",
587
+ "barAlignment": 0,
588
+ "lineWidth": 1,
589
+ "fillOpacity": 8,
590
+ "gradientMode": "none",
591
+ "spanNulls": false,
592
+ "showPoints": "auto",
593
+ "pointSize": 5,
594
+ "stacking": {
595
+ "mode": "none",
596
+ "group": "A"
597
+ },
598
+ "axisPlacement": "auto",
599
+ "axisLabel": "",
600
+ "axisColorMode": "text",
601
+ "scaleDistribution": {
602
+ "type": "linear"
603
+ },
604
+ "hideFrom": {
605
+ "tooltip": false,
606
+ "viz": false,
607
+ "legend": false
608
+ },
609
+ "thresholdsStyle": {
610
+ "mode": "off"
611
+ }
612
+ },
613
+ "thresholds": {
614
+ "mode": "absolute",
615
+ "steps": [
616
+ {
617
+ "color": "green",
618
+ "value": null
619
+ }
620
+ ]
621
+ },
622
+ "mappings": []
623
+ },
624
+ "overrides": []
625
+ },
626
+ "options": {
627
+ "tooltip": {
628
+ "mode": "multi",
629
+ "sort": "desc"
630
+ },
631
+ "legend": {
632
+ "displayMode": "table",
633
+ "placement": "bottom",
634
+ "calcs": [
635
+ "lastNotNull",
636
+ "mean",
637
+ "max"
638
+ ]
639
+ }
640
+ },
641
+ "description": "OpenAI-compatible chat completion HTTP request rates.",
642
+ "id": 8
643
+ },
644
+ {
645
+ "type": "timeseries",
646
+ "title": "TTFT (time to first token)",
647
+ "datasource": {
648
+ "type": "prometheus",
649
+ "uid": "prometheus"
650
+ },
651
+ "gridPos": {
652
+ "x": 0,
653
+ "y": 14,
654
+ "w": 12,
655
+ "h": 8
656
+ },
657
+ "targets": [
658
+ {
659
+ "datasource": {
660
+ "type": "prometheus",
661
+ "uid": "prometheus"
662
+ },
663
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:time_to_first_token_seconds_bucket|ray_vllm_time_to_first_token_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
664
+ "refId": "A",
665
+ "legendFormat": "p50"
666
+ },
667
+ {
668
+ "datasource": {
669
+ "type": "prometheus",
670
+ "uid": "prometheus"
671
+ },
672
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:time_to_first_token_seconds_bucket|ray_vllm_time_to_first_token_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
673
+ "refId": "B",
674
+ "legendFormat": "p95"
675
+ },
676
+ {
677
+ "datasource": {
678
+ "type": "prometheus",
679
+ "uid": "prometheus"
680
+ },
681
+ "expr": "histogram_quantile(0.99, sum by (le) (rate({__name__=~\"vllm:time_to_first_token_seconds_bucket|ray_vllm_time_to_first_token_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
682
+ "refId": "C",
683
+ "legendFormat": "p99"
684
+ }
685
+ ],
686
+ "fieldConfig": {
687
+ "defaults": {
688
+ "unit": "s",
689
+ "color": {
690
+ "mode": "palette-classic"
691
+ },
692
+ "custom": {
693
+ "drawStyle": "line",
694
+ "lineInterpolation": "linear",
695
+ "barAlignment": 0,
696
+ "lineWidth": 1,
697
+ "fillOpacity": 8,
698
+ "gradientMode": "none",
699
+ "spanNulls": false,
700
+ "showPoints": "auto",
701
+ "pointSize": 5,
702
+ "stacking": {
703
+ "mode": "none",
704
+ "group": "A"
705
+ },
706
+ "axisPlacement": "auto",
707
+ "axisLabel": "",
708
+ "axisColorMode": "text",
709
+ "scaleDistribution": {
710
+ "type": "linear"
711
+ },
712
+ "hideFrom": {
713
+ "tooltip": false,
714
+ "viz": false,
715
+ "legend": false
716
+ },
717
+ "thresholdsStyle": {
718
+ "mode": "off"
719
+ }
720
+ },
721
+ "thresholds": {
722
+ "mode": "absolute",
723
+ "steps": [
724
+ {
725
+ "color": "green",
726
+ "value": null
727
+ }
728
+ ]
729
+ },
730
+ "mappings": []
731
+ },
732
+ "overrides": []
733
+ },
734
+ "options": {
735
+ "tooltip": {
736
+ "mode": "multi",
737
+ "sort": "desc"
738
+ },
739
+ "legend": {
740
+ "displayMode": "table",
741
+ "placement": "bottom",
742
+ "calcs": [
743
+ "lastNotNull",
744
+ "mean",
745
+ "max"
746
+ ]
747
+ }
748
+ },
749
+ "description": "Time to first token quantiles from vLLM histogram metrics.",
750
+ "id": 9
751
+ },
752
+ {
753
+ "type": "timeseries",
754
+ "title": "End-to-end request latency",
755
+ "datasource": {
756
+ "type": "prometheus",
757
+ "uid": "prometheus"
758
+ },
759
+ "gridPos": {
760
+ "x": 12,
761
+ "y": 14,
762
+ "w": 12,
763
+ "h": 8
764
+ },
765
+ "targets": [
766
+ {
767
+ "datasource": {
768
+ "type": "prometheus",
769
+ "uid": "prometheus"
770
+ },
771
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:e2e_request_latency_seconds_bucket|ray_vllm_e2e_request_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
772
+ "refId": "A",
773
+ "legendFormat": "p50"
774
+ },
775
+ {
776
+ "datasource": {
777
+ "type": "prometheus",
778
+ "uid": "prometheus"
779
+ },
780
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:e2e_request_latency_seconds_bucket|ray_vllm_e2e_request_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
781
+ "refId": "B",
782
+ "legendFormat": "p95"
783
+ },
784
+ {
785
+ "datasource": {
786
+ "type": "prometheus",
787
+ "uid": "prometheus"
788
+ },
789
+ "expr": "histogram_quantile(0.99, sum by (le) (rate({__name__=~\"vllm:e2e_request_latency_seconds_bucket|ray_vllm_e2e_request_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
790
+ "refId": "C",
791
+ "legendFormat": "p99"
792
+ }
793
+ ],
794
+ "fieldConfig": {
795
+ "defaults": {
796
+ "unit": "s",
797
+ "color": {
798
+ "mode": "palette-classic"
799
+ },
800
+ "custom": {
801
+ "drawStyle": "line",
802
+ "lineInterpolation": "linear",
803
+ "barAlignment": 0,
804
+ "lineWidth": 1,
805
+ "fillOpacity": 8,
806
+ "gradientMode": "none",
807
+ "spanNulls": false,
808
+ "showPoints": "auto",
809
+ "pointSize": 5,
810
+ "stacking": {
811
+ "mode": "none",
812
+ "group": "A"
813
+ },
814
+ "axisPlacement": "auto",
815
+ "axisLabel": "",
816
+ "axisColorMode": "text",
817
+ "scaleDistribution": {
818
+ "type": "linear"
819
+ },
820
+ "hideFrom": {
821
+ "tooltip": false,
822
+ "viz": false,
823
+ "legend": false
824
+ },
825
+ "thresholdsStyle": {
826
+ "mode": "off"
827
+ }
828
+ },
829
+ "thresholds": {
830
+ "mode": "absolute",
831
+ "steps": [
832
+ {
833
+ "color": "green",
834
+ "value": null
835
+ }
836
+ ]
837
+ },
838
+ "mappings": []
839
+ },
840
+ "overrides": []
841
+ },
842
+ "options": {
843
+ "tooltip": {
844
+ "mode": "multi",
845
+ "sort": "desc"
846
+ },
847
+ "legend": {
848
+ "displayMode": "table",
849
+ "placement": "bottom",
850
+ "calcs": [
851
+ "lastNotNull",
852
+ "mean",
853
+ "max"
854
+ ]
855
+ }
856
+ },
857
+ "id": 10
858
+ },
859
+ {
860
+ "type": "timeseries",
861
+ "title": "Inter-token latency",
862
+ "datasource": {
863
+ "type": "prometheus",
864
+ "uid": "prometheus"
865
+ },
866
+ "gridPos": {
867
+ "x": 0,
868
+ "y": 22,
869
+ "w": 12,
870
+ "h": 8
871
+ },
872
+ "targets": [
873
+ {
874
+ "datasource": {
875
+ "type": "prometheus",
876
+ "uid": "prometheus"
877
+ },
878
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:inter_token_latency_seconds_bucket|ray_vllm_inter_token_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
879
+ "refId": "A",
880
+ "legendFormat": "p50"
881
+ },
882
+ {
883
+ "datasource": {
884
+ "type": "prometheus",
885
+ "uid": "prometheus"
886
+ },
887
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:inter_token_latency_seconds_bucket|ray_vllm_inter_token_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
888
+ "refId": "B",
889
+ "legendFormat": "p95"
890
+ },
891
+ {
892
+ "datasource": {
893
+ "type": "prometheus",
894
+ "uid": "prometheus"
895
+ },
896
+ "expr": "histogram_quantile(0.99, sum by (le) (rate({__name__=~\"vllm:inter_token_latency_seconds_bucket|ray_vllm_inter_token_latency_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
897
+ "refId": "C",
898
+ "legendFormat": "p99"
899
+ }
900
+ ],
901
+ "fieldConfig": {
902
+ "defaults": {
903
+ "unit": "s",
904
+ "color": {
905
+ "mode": "palette-classic"
906
+ },
907
+ "custom": {
908
+ "drawStyle": "line",
909
+ "lineInterpolation": "linear",
910
+ "barAlignment": 0,
911
+ "lineWidth": 1,
912
+ "fillOpacity": 8,
913
+ "gradientMode": "none",
914
+ "spanNulls": false,
915
+ "showPoints": "auto",
916
+ "pointSize": 5,
917
+ "stacking": {
918
+ "mode": "none",
919
+ "group": "A"
920
+ },
921
+ "axisPlacement": "auto",
922
+ "axisLabel": "",
923
+ "axisColorMode": "text",
924
+ "scaleDistribution": {
925
+ "type": "linear"
926
+ },
927
+ "hideFrom": {
928
+ "tooltip": false,
929
+ "viz": false,
930
+ "legend": false
931
+ },
932
+ "thresholdsStyle": {
933
+ "mode": "off"
934
+ }
935
+ },
936
+ "thresholds": {
937
+ "mode": "absolute",
938
+ "steps": [
939
+ {
940
+ "color": "green",
941
+ "value": null
942
+ }
943
+ ]
944
+ },
945
+ "mappings": []
946
+ },
947
+ "overrides": []
948
+ },
949
+ "options": {
950
+ "tooltip": {
951
+ "mode": "multi",
952
+ "sort": "desc"
953
+ },
954
+ "legend": {
955
+ "displayMode": "table",
956
+ "placement": "bottom",
957
+ "calcs": [
958
+ "lastNotNull",
959
+ "mean",
960
+ "max"
961
+ ]
962
+ }
963
+ },
964
+ "description": "Inter-token latency; useful for perceived streaming speed.",
965
+ "id": 11
966
+ },
967
+ {
968
+ "type": "timeseries",
969
+ "title": "Queue / prefill latency",
970
+ "datasource": {
971
+ "type": "prometheus",
972
+ "uid": "prometheus"
973
+ },
974
+ "gridPos": {
975
+ "x": 12,
976
+ "y": 22,
977
+ "w": 12,
978
+ "h": 8
979
+ },
980
+ "targets": [
981
+ {
982
+ "datasource": {
983
+ "type": "prometheus",
984
+ "uid": "prometheus"
985
+ },
986
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:request_queue_time_seconds_bucket|ray_vllm_request_queue_time_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
987
+ "refId": "A",
988
+ "legendFormat": "queue p95"
989
+ },
990
+ {
991
+ "datasource": {
992
+ "type": "prometheus",
993
+ "uid": "prometheus"
994
+ },
995
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:request_prefill_time_seconds_bucket|ray_vllm_request_prefill_time_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])))",
996
+ "refId": "B",
997
+ "legendFormat": "prefill p95"
998
+ }
999
+ ],
1000
+ "fieldConfig": {
1001
+ "defaults": {
1002
+ "unit": "s",
1003
+ "color": {
1004
+ "mode": "palette-classic"
1005
+ },
1006
+ "custom": {
1007
+ "drawStyle": "line",
1008
+ "lineInterpolation": "linear",
1009
+ "barAlignment": 0,
1010
+ "lineWidth": 1,
1011
+ "fillOpacity": 8,
1012
+ "gradientMode": "none",
1013
+ "spanNulls": false,
1014
+ "showPoints": "auto",
1015
+ "pointSize": 5,
1016
+ "stacking": {
1017
+ "mode": "none",
1018
+ "group": "A"
1019
+ },
1020
+ "axisPlacement": "auto",
1021
+ "axisLabel": "",
1022
+ "axisColorMode": "text",
1023
+ "scaleDistribution": {
1024
+ "type": "linear"
1025
+ },
1026
+ "hideFrom": {
1027
+ "tooltip": false,
1028
+ "viz": false,
1029
+ "legend": false
1030
+ },
1031
+ "thresholdsStyle": {
1032
+ "mode": "off"
1033
+ }
1034
+ },
1035
+ "thresholds": {
1036
+ "mode": "absolute",
1037
+ "steps": [
1038
+ {
1039
+ "color": "green",
1040
+ "value": null
1041
+ }
1042
+ ]
1043
+ },
1044
+ "mappings": []
1045
+ },
1046
+ "overrides": []
1047
+ },
1048
+ "options": {
1049
+ "tooltip": {
1050
+ "mode": "multi",
1051
+ "sort": "desc"
1052
+ },
1053
+ "legend": {
1054
+ "displayMode": "table",
1055
+ "placement": "bottom",
1056
+ "calcs": [
1057
+ "lastNotNull",
1058
+ "mean",
1059
+ "max"
1060
+ ]
1061
+ }
1062
+ },
1063
+ "id": 12
1064
+ },
1065
+ {
1066
+ "type": "timeseries",
1067
+ "title": "MTP efficiency",
1068
+ "datasource": {
1069
+ "type": "prometheus",
1070
+ "uid": "prometheus"
1071
+ },
1072
+ "gridPos": {
1073
+ "x": 0,
1074
+ "y": 30,
1075
+ "w": 12,
1076
+ "h": 8
1077
+ },
1078
+ "targets": [
1079
+ {
1080
+ "datasource": {
1081
+ "type": "prometheus",
1082
+ "uid": "prometheus"
1083
+ },
1084
+ "expr": "100 * sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_total|ray_vllm_spec_decode_num_accepted_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_draft_tokens_total|ray_vllm_spec_decode_num_draft_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1085
+ "refId": "A",
1086
+ "legendFormat": "acceptance %"
1087
+ },
1088
+ {
1089
+ "datasource": {
1090
+ "type": "prometheus",
1091
+ "uid": "prometheus"
1092
+ },
1093
+ "expr": "sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_total|ray_vllm_spec_decode_num_accepted_tokens_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_drafts_total|ray_vllm_spec_decode_num_drafts_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1094
+ "refId": "B",
1095
+ "legendFormat": "accepted tokens / draft"
1096
+ }
1097
+ ],
1098
+ "fieldConfig": {
1099
+ "defaults": {
1100
+ "unit": "short",
1101
+ "color": {
1102
+ "mode": "palette-classic"
1103
+ },
1104
+ "custom": {
1105
+ "drawStyle": "line",
1106
+ "lineInterpolation": "linear",
1107
+ "barAlignment": 0,
1108
+ "lineWidth": 1,
1109
+ "fillOpacity": 8,
1110
+ "gradientMode": "none",
1111
+ "spanNulls": false,
1112
+ "showPoints": "auto",
1113
+ "pointSize": 5,
1114
+ "stacking": {
1115
+ "mode": "none",
1116
+ "group": "A"
1117
+ },
1118
+ "axisPlacement": "auto",
1119
+ "axisLabel": "",
1120
+ "axisColorMode": "text",
1121
+ "scaleDistribution": {
1122
+ "type": "linear"
1123
+ },
1124
+ "hideFrom": {
1125
+ "tooltip": false,
1126
+ "viz": false,
1127
+ "legend": false
1128
+ },
1129
+ "thresholdsStyle": {
1130
+ "mode": "off"
1131
+ }
1132
+ },
1133
+ "thresholds": {
1134
+ "mode": "absolute",
1135
+ "steps": [
1136
+ {
1137
+ "color": "green",
1138
+ "value": null
1139
+ }
1140
+ ]
1141
+ },
1142
+ "mappings": []
1143
+ },
1144
+ "overrides": []
1145
+ },
1146
+ "options": {
1147
+ "tooltip": {
1148
+ "mode": "multi",
1149
+ "sort": "desc"
1150
+ },
1151
+ "legend": {
1152
+ "displayMode": "table",
1153
+ "placement": "bottom",
1154
+ "calcs": [
1155
+ "lastNotNull",
1156
+ "mean",
1157
+ "max"
1158
+ ]
1159
+ }
1160
+ },
1161
+ "description": "Overall MTP token acceptance and mean accepted speculative tokens per draft.",
1162
+ "id": 13
1163
+ },
1164
+ {
1165
+ "type": "timeseries",
1166
+ "title": "MTP acceptance by draft position",
1167
+ "datasource": {
1168
+ "type": "prometheus",
1169
+ "uid": "prometheus"
1170
+ },
1171
+ "gridPos": {
1172
+ "x": 12,
1173
+ "y": 30,
1174
+ "w": 12,
1175
+ "h": 8
1176
+ },
1177
+ "targets": [
1178
+ {
1179
+ "datasource": {
1180
+ "type": "prometheus",
1181
+ "uid": "prometheus"
1182
+ },
1183
+ "expr": "100 * sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_per_pos_total|ray_vllm_spec_decode_num_accepted_tokens_per_pos_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",position=\"0\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_drafts_total|ray_vllm_spec_decode_num_drafts_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1184
+ "refId": "A",
1185
+ "legendFormat": "position 0"
1186
+ },
1187
+ {
1188
+ "datasource": {
1189
+ "type": "prometheus",
1190
+ "uid": "prometheus"
1191
+ },
1192
+ "expr": "100 * sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_per_pos_total|ray_vllm_spec_decode_num_accepted_tokens_per_pos_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",position=\"1\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_drafts_total|ray_vllm_spec_decode_num_drafts_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1193
+ "refId": "B",
1194
+ "legendFormat": "position 1"
1195
+ },
1196
+ {
1197
+ "datasource": {
1198
+ "type": "prometheus",
1199
+ "uid": "prometheus"
1200
+ },
1201
+ "expr": "100 * sum(rate({__name__=~\"vllm:spec_decode_num_accepted_tokens_per_pos_total|ray_vllm_spec_decode_num_accepted_tokens_per_pos_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",position=\"2\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:spec_decode_num_drafts_total|ray_vllm_spec_decode_num_drafts_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1202
+ "refId": "C",
1203
+ "legendFormat": "position 2"
1204
+ }
1205
+ ],
1206
+ "fieldConfig": {
1207
+ "defaults": {
1208
+ "unit": "percent",
1209
+ "color": {
1210
+ "mode": "palette-classic"
1211
+ },
1212
+ "custom": {
1213
+ "drawStyle": "line",
1214
+ "lineInterpolation": "linear",
1215
+ "barAlignment": 0,
1216
+ "lineWidth": 1,
1217
+ "fillOpacity": 8,
1218
+ "gradientMode": "none",
1219
+ "spanNulls": false,
1220
+ "showPoints": "auto",
1221
+ "pointSize": 5,
1222
+ "stacking": {
1223
+ "mode": "none",
1224
+ "group": "A"
1225
+ },
1226
+ "axisPlacement": "auto",
1227
+ "axisLabel": "",
1228
+ "axisColorMode": "text",
1229
+ "scaleDistribution": {
1230
+ "type": "linear"
1231
+ },
1232
+ "hideFrom": {
1233
+ "tooltip": false,
1234
+ "viz": false,
1235
+ "legend": false
1236
+ },
1237
+ "thresholdsStyle": {
1238
+ "mode": "off"
1239
+ }
1240
+ },
1241
+ "thresholds": {
1242
+ "mode": "absolute",
1243
+ "steps": [
1244
+ {
1245
+ "color": "green",
1246
+ "value": null
1247
+ }
1248
+ ]
1249
+ },
1250
+ "mappings": [],
1251
+ "min": 0,
1252
+ "max": 100
1253
+ },
1254
+ "overrides": []
1255
+ },
1256
+ "options": {
1257
+ "tooltip": {
1258
+ "mode": "multi",
1259
+ "sort": "desc"
1260
+ },
1261
+ "legend": {
1262
+ "displayMode": "table",
1263
+ "placement": "bottom",
1264
+ "calcs": [
1265
+ "lastNotNull",
1266
+ "mean",
1267
+ "max"
1268
+ ]
1269
+ }
1270
+ },
1271
+ "description": "Unconditional acceptance probability for each MTP draft position. With MTP3 this shows whether later speculative positions still contribute.",
1272
+ "id": 14
1273
+ },
1274
+ {
1275
+ "type": "timeseries",
1276
+ "title": "Cache behavior",
1277
+ "datasource": {
1278
+ "type": "prometheus",
1279
+ "uid": "prometheus"
1280
+ },
1281
+ "gridPos": {
1282
+ "x": 0,
1283
+ "y": 38,
1284
+ "w": 12,
1285
+ "h": 8
1286
+ },
1287
+ "targets": [
1288
+ {
1289
+ "datasource": {
1290
+ "type": "prometheus",
1291
+ "uid": "prometheus"
1292
+ },
1293
+ "expr": "100 * sum(rate({__name__=~\"vllm:prefix_cache_hits_total|ray_vllm_prefix_cache_hits_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) / clamp_min(sum(rate({__name__=~\"vllm:prefix_cache_queries_total|ray_vllm_prefix_cache_queries_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])), 0.000001)",
1294
+ "refId": "A",
1295
+ "legendFormat": "prefix hit %"
1296
+ },
1297
+ {
1298
+ "datasource": {
1299
+ "type": "prometheus",
1300
+ "uid": "prometheus"
1301
+ },
1302
+ "expr": "(100 * avg(last_over_time({__name__=~\"vllm:kv_cache_usage_perc|ray_vllm_kv_cache_usage_perc\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1303
+ "refId": "B",
1304
+ "legendFormat": "KV usage %"
1305
+ }
1306
+ ],
1307
+ "fieldConfig": {
1308
+ "defaults": {
1309
+ "unit": "percent",
1310
+ "color": {
1311
+ "mode": "palette-classic"
1312
+ },
1313
+ "custom": {
1314
+ "drawStyle": "line",
1315
+ "lineInterpolation": "linear",
1316
+ "barAlignment": 0,
1317
+ "lineWidth": 1,
1318
+ "fillOpacity": 8,
1319
+ "gradientMode": "none",
1320
+ "spanNulls": false,
1321
+ "showPoints": "auto",
1322
+ "pointSize": 5,
1323
+ "stacking": {
1324
+ "mode": "none",
1325
+ "group": "A"
1326
+ },
1327
+ "axisPlacement": "auto",
1328
+ "axisLabel": "",
1329
+ "axisColorMode": "text",
1330
+ "scaleDistribution": {
1331
+ "type": "linear"
1332
+ },
1333
+ "hideFrom": {
1334
+ "tooltip": false,
1335
+ "viz": false,
1336
+ "legend": false
1337
+ },
1338
+ "thresholdsStyle": {
1339
+ "mode": "off"
1340
+ }
1341
+ },
1342
+ "thresholds": {
1343
+ "mode": "absolute",
1344
+ "steps": [
1345
+ {
1346
+ "color": "green",
1347
+ "value": null
1348
+ }
1349
+ ]
1350
+ },
1351
+ "mappings": [],
1352
+ "min": 0,
1353
+ "max": 100
1354
+ },
1355
+ "overrides": []
1356
+ },
1357
+ "options": {
1358
+ "tooltip": {
1359
+ "mode": "multi",
1360
+ "sort": "desc"
1361
+ },
1362
+ "legend": {
1363
+ "displayMode": "table",
1364
+ "placement": "bottom",
1365
+ "calcs": [
1366
+ "lastNotNull",
1367
+ "mean",
1368
+ "max"
1369
+ ]
1370
+ }
1371
+ },
1372
+ "id": 15
1373
+ },
1374
+ {
1375
+ "type": "timeseries",
1376
+ "title": "Request outcomes",
1377
+ "datasource": {
1378
+ "type": "prometheus",
1379
+ "uid": "prometheus"
1380
+ },
1381
+ "gridPos": {
1382
+ "x": 12,
1383
+ "y": 38,
1384
+ "w": 12,
1385
+ "h": 8
1386
+ },
1387
+ "targets": [
1388
+ {
1389
+ "datasource": {
1390
+ "type": "prometheus",
1391
+ "uid": "prometheus"
1392
+ },
1393
+ "expr": "(sum(rate({__name__=~\"vllm:request_success_total|ray_vllm_request_success_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",finished_reason=\"stop\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1394
+ "refId": "A",
1395
+ "legendFormat": "stop"
1396
+ },
1397
+ {
1398
+ "datasource": {
1399
+ "type": "prometheus",
1400
+ "uid": "prometheus"
1401
+ },
1402
+ "expr": "(sum(rate({__name__=~\"vllm:request_success_total|ray_vllm_request_success_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",finished_reason=\"length\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1403
+ "refId": "B",
1404
+ "legendFormat": "length"
1405
+ },
1406
+ {
1407
+ "datasource": {
1408
+ "type": "prometheus",
1409
+ "uid": "prometheus"
1410
+ },
1411
+ "expr": "(sum(rate({__name__=~\"vllm:request_success_total|ray_vllm_request_success_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",finished_reason=\"abort\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1412
+ "refId": "C",
1413
+ "legendFormat": "abort"
1414
+ },
1415
+ {
1416
+ "datasource": {
1417
+ "type": "prometheus",
1418
+ "uid": "prometheus"
1419
+ },
1420
+ "expr": "(sum(rate({__name__=~\"vllm:request_success_total|ray_vllm_request_success_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",finished_reason=\"error\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1421
+ "refId": "D",
1422
+ "legendFormat": "error"
1423
+ },
1424
+ {
1425
+ "datasource": {
1426
+ "type": "prometheus",
1427
+ "uid": "prometheus"
1428
+ },
1429
+ "expr": "(sum(rate({__name__=~\"vllm:request_success_total|ray_vllm_request_success_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",finished_reason=\"repetition\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1430
+ "refId": "E",
1431
+ "legendFormat": "repetition"
1432
+ }
1433
+ ],
1434
+ "fieldConfig": {
1435
+ "defaults": {
1436
+ "unit": "short",
1437
+ "color": {
1438
+ "mode": "palette-classic"
1439
+ },
1440
+ "custom": {
1441
+ "drawStyle": "line",
1442
+ "lineInterpolation": "linear",
1443
+ "barAlignment": 0,
1444
+ "lineWidth": 1,
1445
+ "fillOpacity": 8,
1446
+ "gradientMode": "none",
1447
+ "spanNulls": false,
1448
+ "showPoints": "auto",
1449
+ "pointSize": 5,
1450
+ "stacking": {
1451
+ "mode": "none",
1452
+ "group": "A"
1453
+ },
1454
+ "axisPlacement": "auto",
1455
+ "axisLabel": "",
1456
+ "axisColorMode": "text",
1457
+ "scaleDistribution": {
1458
+ "type": "linear"
1459
+ },
1460
+ "hideFrom": {
1461
+ "tooltip": false,
1462
+ "viz": false,
1463
+ "legend": false
1464
+ },
1465
+ "thresholdsStyle": {
1466
+ "mode": "off"
1467
+ }
1468
+ },
1469
+ "thresholds": {
1470
+ "mode": "absolute",
1471
+ "steps": [
1472
+ {
1473
+ "color": "green",
1474
+ "value": null
1475
+ }
1476
+ ]
1477
+ },
1478
+ "mappings": []
1479
+ },
1480
+ "overrides": []
1481
+ },
1482
+ "options": {
1483
+ "tooltip": {
1484
+ "mode": "multi",
1485
+ "sort": "desc"
1486
+ },
1487
+ "legend": {
1488
+ "displayMode": "table",
1489
+ "placement": "bottom",
1490
+ "calcs": [
1491
+ "lastNotNull",
1492
+ "mean",
1493
+ "max"
1494
+ ]
1495
+ }
1496
+ },
1497
+ "description": "Rate of finished requests by vLLM finish reason.",
1498
+ "id": 16
1499
+ },
1500
+ {
1501
+ "type": "timeseries",
1502
+ "title": "Scheduler pressure",
1503
+ "datasource": {
1504
+ "type": "prometheus",
1505
+ "uid": "prometheus"
1506
+ },
1507
+ "gridPos": {
1508
+ "x": 0,
1509
+ "y": 46,
1510
+ "w": 12,
1511
+ "h": 8
1512
+ },
1513
+ "targets": [
1514
+ {
1515
+ "datasource": {
1516
+ "type": "prometheus",
1517
+ "uid": "prometheus"
1518
+ },
1519
+ "expr": "(sum(last_over_time({__name__=~\"vllm:num_requests_running|ray_vllm_num_requests_running\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1520
+ "refId": "A",
1521
+ "legendFormat": "running"
1522
+ },
1523
+ {
1524
+ "datasource": {
1525
+ "type": "prometheus",
1526
+ "uid": "prometheus"
1527
+ },
1528
+ "expr": "(sum(last_over_time({__name__=~\"vllm:num_requests_waiting|ray_vllm_num_requests_waiting\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1529
+ "refId": "B",
1530
+ "legendFormat": "waiting"
1531
+ },
1532
+ {
1533
+ "datasource": {
1534
+ "type": "prometheus",
1535
+ "uid": "prometheus"
1536
+ },
1537
+ "expr": "(sum(rate({__name__=~\"vllm:num_preemptions_total|ray_vllm_num_preemptions_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[5m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1538
+ "refId": "C",
1539
+ "legendFormat": "preemptions/s"
1540
+ }
1541
+ ],
1542
+ "fieldConfig": {
1543
+ "defaults": {
1544
+ "unit": "short",
1545
+ "color": {
1546
+ "mode": "palette-classic"
1547
+ },
1548
+ "custom": {
1549
+ "drawStyle": "line",
1550
+ "lineInterpolation": "linear",
1551
+ "barAlignment": 0,
1552
+ "lineWidth": 1,
1553
+ "fillOpacity": 8,
1554
+ "gradientMode": "none",
1555
+ "spanNulls": false,
1556
+ "showPoints": "auto",
1557
+ "pointSize": 5,
1558
+ "stacking": {
1559
+ "mode": "none",
1560
+ "group": "A"
1561
+ },
1562
+ "axisPlacement": "auto",
1563
+ "axisLabel": "",
1564
+ "axisColorMode": "text",
1565
+ "scaleDistribution": {
1566
+ "type": "linear"
1567
+ },
1568
+ "hideFrom": {
1569
+ "tooltip": false,
1570
+ "viz": false,
1571
+ "legend": false
1572
+ },
1573
+ "thresholdsStyle": {
1574
+ "mode": "off"
1575
+ }
1576
+ },
1577
+ "thresholds": {
1578
+ "mode": "absolute",
1579
+ "steps": [
1580
+ {
1581
+ "color": "green",
1582
+ "value": null
1583
+ }
1584
+ ]
1585
+ },
1586
+ "mappings": []
1587
+ },
1588
+ "overrides": []
1589
+ },
1590
+ "options": {
1591
+ "tooltip": {
1592
+ "mode": "multi",
1593
+ "sort": "desc"
1594
+ },
1595
+ "legend": {
1596
+ "displayMode": "table",
1597
+ "placement": "bottom",
1598
+ "calcs": [
1599
+ "lastNotNull",
1600
+ "mean",
1601
+ "max"
1602
+ ]
1603
+ }
1604
+ },
1605
+ "description": "Running/waiting requests and preemption activity.",
1606
+ "id": 17
1607
+ },
1608
+ {
1609
+ "type": "timeseries",
1610
+ "title": "Prompt token sources",
1611
+ "datasource": {
1612
+ "type": "prometheus",
1613
+ "uid": "prometheus"
1614
+ },
1615
+ "gridPos": {
1616
+ "x": 12,
1617
+ "y": 46,
1618
+ "w": 12,
1619
+ "h": 8
1620
+ },
1621
+ "targets": [
1622
+ {
1623
+ "datasource": {
1624
+ "type": "prometheus",
1625
+ "uid": "prometheus"
1626
+ },
1627
+ "expr": "(sum(rate({__name__=~\"vllm:prompt_tokens_by_source_total|ray_vllm_prompt_tokens_by_source_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",source=\"local_compute\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1628
+ "refId": "A",
1629
+ "legendFormat": "local compute"
1630
+ },
1631
+ {
1632
+ "datasource": {
1633
+ "type": "prometheus",
1634
+ "uid": "prometheus"
1635
+ },
1636
+ "expr": "(sum(rate({__name__=~\"vllm:prompt_tokens_by_source_total|ray_vllm_prompt_tokens_by_source_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",source=\"local_cache_hit\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1637
+ "refId": "B",
1638
+ "legendFormat": "local cache hit"
1639
+ },
1640
+ {
1641
+ "datasource": {
1642
+ "type": "prometheus",
1643
+ "uid": "prometheus"
1644
+ },
1645
+ "expr": "(sum(rate({__name__=~\"vllm:prompt_tokens_by_source_total|ray_vllm_prompt_tokens_by_source_total\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\",source=\"external_kv_transfer\"}[1m])) or (0 * sum(rate({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}[5m]))))",
1646
+ "refId": "C",
1647
+ "legendFormat": "external KV"
1648
+ }
1649
+ ],
1650
+ "fieldConfig": {
1651
+ "defaults": {
1652
+ "unit": "short",
1653
+ "color": {
1654
+ "mode": "palette-classic"
1655
+ },
1656
+ "custom": {
1657
+ "drawStyle": "line",
1658
+ "lineInterpolation": "linear",
1659
+ "barAlignment": 0,
1660
+ "lineWidth": 1,
1661
+ "fillOpacity": 8,
1662
+ "gradientMode": "none",
1663
+ "spanNulls": false,
1664
+ "showPoints": "auto",
1665
+ "pointSize": 5,
1666
+ "stacking": {
1667
+ "mode": "none",
1668
+ "group": "A"
1669
+ },
1670
+ "axisPlacement": "auto",
1671
+ "axisLabel": "",
1672
+ "axisColorMode": "text",
1673
+ "scaleDistribution": {
1674
+ "type": "linear"
1675
+ },
1676
+ "hideFrom": {
1677
+ "tooltip": false,
1678
+ "viz": false,
1679
+ "legend": false
1680
+ },
1681
+ "thresholdsStyle": {
1682
+ "mode": "off"
1683
+ }
1684
+ },
1685
+ "thresholds": {
1686
+ "mode": "absolute",
1687
+ "steps": [
1688
+ {
1689
+ "color": "green",
1690
+ "value": null
1691
+ }
1692
+ ]
1693
+ },
1694
+ "mappings": []
1695
+ },
1696
+ "overrides": []
1697
+ },
1698
+ "options": {
1699
+ "tooltip": {
1700
+ "mode": "multi",
1701
+ "sort": "desc"
1702
+ },
1703
+ "legend": {
1704
+ "displayMode": "table",
1705
+ "placement": "bottom",
1706
+ "calcs": [
1707
+ "lastNotNull",
1708
+ "mean",
1709
+ "max"
1710
+ ]
1711
+ }
1712
+ },
1713
+ "description": "Prompt tokens by source. Your current endpoint exposes local compute, local cache hit, and external KV transfer.",
1714
+ "id": 18
1715
+ },
1716
+ {
1717
+ "type": "timeseries",
1718
+ "title": "KV block lifetime",
1719
+ "datasource": {
1720
+ "type": "prometheus",
1721
+ "uid": "prometheus"
1722
+ },
1723
+ "gridPos": {
1724
+ "h": 8,
1725
+ "w": 8,
1726
+ "x": 0,
1727
+ "y": 54
1728
+ },
1729
+ "targets": [
1730
+ {
1731
+ "datasource": {
1732
+ "type": "prometheus",
1733
+ "uid": "prometheus"
1734
+ },
1735
+ "editorMode": "code",
1736
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:kv_block_lifetime_seconds_bucket|ray_vllm_kv_block_lifetime_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1737
+ "legendFormat": "p50",
1738
+ "range": true,
1739
+ "refId": "A"
1740
+ },
1741
+ {
1742
+ "datasource": {
1743
+ "type": "prometheus",
1744
+ "uid": "prometheus"
1745
+ },
1746
+ "editorMode": "code",
1747
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:kv_block_lifetime_seconds_bucket|ray_vllm_kv_block_lifetime_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1748
+ "legendFormat": "p95",
1749
+ "range": true,
1750
+ "refId": "B"
1751
+ }
1752
+ ],
1753
+ "fieldConfig": {
1754
+ "defaults": {
1755
+ "unit": "s",
1756
+ "color": {
1757
+ "mode": "palette-classic"
1758
+ },
1759
+ "custom": {
1760
+ "drawStyle": "line",
1761
+ "lineInterpolation": "linear",
1762
+ "barAlignment": 0,
1763
+ "lineWidth": 1,
1764
+ "fillOpacity": 8,
1765
+ "gradientMode": "none",
1766
+ "spanNulls": false,
1767
+ "showPoints": "auto",
1768
+ "pointSize": 5,
1769
+ "stacking": {
1770
+ "mode": "none",
1771
+ "group": "A"
1772
+ },
1773
+ "axisPlacement": "auto",
1774
+ "axisLabel": "",
1775
+ "axisColorMode": "text",
1776
+ "scaleDistribution": {
1777
+ "type": "linear"
1778
+ },
1779
+ "hideFrom": {
1780
+ "tooltip": false,
1781
+ "viz": false,
1782
+ "legend": false
1783
+ },
1784
+ "thresholdsStyle": {
1785
+ "mode": "off"
1786
+ }
1787
+ },
1788
+ "thresholds": {
1789
+ "mode": "absolute",
1790
+ "steps": [
1791
+ {
1792
+ "color": "green",
1793
+ "value": null
1794
+ }
1795
+ ]
1796
+ },
1797
+ "mappings": []
1798
+ },
1799
+ "overrides": []
1800
+ },
1801
+ "options": {
1802
+ "tooltip": {
1803
+ "mode": "multi",
1804
+ "sort": "desc"
1805
+ },
1806
+ "legend": {
1807
+ "displayMode": "table",
1808
+ "placement": "bottom",
1809
+ "calcs": [
1810
+ "lastNotNull",
1811
+ "mean",
1812
+ "max"
1813
+ ]
1814
+ }
1815
+ },
1816
+ "description": "How long a cache block survives from allocation to eviction. Short lifetimes next to a low prefix-cache hit rate mean the cache is being churned faster than it can be reused.",
1817
+ "id": 19
1818
+ },
1819
+ {
1820
+ "type": "timeseries",
1821
+ "title": "KV block reuse gap",
1822
+ "datasource": {
1823
+ "type": "prometheus",
1824
+ "uid": "prometheus"
1825
+ },
1826
+ "gridPos": {
1827
+ "h": 8,
1828
+ "w": 8,
1829
+ "x": 8,
1830
+ "y": 54
1831
+ },
1832
+ "targets": [
1833
+ {
1834
+ "datasource": {
1835
+ "type": "prometheus",
1836
+ "uid": "prometheus"
1837
+ },
1838
+ "editorMode": "code",
1839
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:kv_block_reuse_gap_seconds_bucket|ray_vllm_kv_block_reuse_gap_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1840
+ "legendFormat": "p50",
1841
+ "range": true,
1842
+ "refId": "A"
1843
+ },
1844
+ {
1845
+ "datasource": {
1846
+ "type": "prometheus",
1847
+ "uid": "prometheus"
1848
+ },
1849
+ "editorMode": "code",
1850
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:kv_block_reuse_gap_seconds_bucket|ray_vllm_kv_block_reuse_gap_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1851
+ "legendFormat": "p95",
1852
+ "range": true,
1853
+ "refId": "B"
1854
+ }
1855
+ ],
1856
+ "fieldConfig": {
1857
+ "defaults": {
1858
+ "unit": "s",
1859
+ "color": {
1860
+ "mode": "palette-classic"
1861
+ },
1862
+ "custom": {
1863
+ "drawStyle": "line",
1864
+ "lineInterpolation": "linear",
1865
+ "barAlignment": 0,
1866
+ "lineWidth": 1,
1867
+ "fillOpacity": 8,
1868
+ "gradientMode": "none",
1869
+ "spanNulls": false,
1870
+ "showPoints": "auto",
1871
+ "pointSize": 5,
1872
+ "stacking": {
1873
+ "mode": "none",
1874
+ "group": "A"
1875
+ },
1876
+ "axisPlacement": "auto",
1877
+ "axisLabel": "",
1878
+ "axisColorMode": "text",
1879
+ "scaleDistribution": {
1880
+ "type": "linear"
1881
+ },
1882
+ "hideFrom": {
1883
+ "tooltip": false,
1884
+ "viz": false,
1885
+ "legend": false
1886
+ },
1887
+ "thresholdsStyle": {
1888
+ "mode": "off"
1889
+ }
1890
+ },
1891
+ "thresholds": {
1892
+ "mode": "absolute",
1893
+ "steps": [
1894
+ {
1895
+ "color": "green",
1896
+ "value": null
1897
+ }
1898
+ ]
1899
+ },
1900
+ "mappings": []
1901
+ },
1902
+ "overrides": []
1903
+ },
1904
+ "options": {
1905
+ "tooltip": {
1906
+ "mode": "multi",
1907
+ "sort": "desc"
1908
+ },
1909
+ "legend": {
1910
+ "displayMode": "table",
1911
+ "placement": "bottom",
1912
+ "calcs": [
1913
+ "lastNotNull",
1914
+ "mean",
1915
+ "max"
1916
+ ]
1917
+ }
1918
+ },
1919
+ "description": "The gap between a block being written and read again. Long gaps with frequent eviction are the signature of a cache that is one size too small for the traffic.",
1920
+ "id": 20
1921
+ },
1922
+ {
1923
+ "type": "timeseries",
1924
+ "title": "KV block idle before evict",
1925
+ "datasource": {
1926
+ "type": "prometheus",
1927
+ "uid": "prometheus"
1928
+ },
1929
+ "gridPos": {
1930
+ "h": 8,
1931
+ "w": 8,
1932
+ "x": 16,
1933
+ "y": 54
1934
+ },
1935
+ "targets": [
1936
+ {
1937
+ "datasource": {
1938
+ "type": "prometheus",
1939
+ "uid": "prometheus"
1940
+ },
1941
+ "editorMode": "code",
1942
+ "expr": "histogram_quantile(0.5, sum by (le) (rate({__name__=~\"vllm:kv_block_idle_before_evict_seconds_bucket|ray_vllm_kv_block_idle_before_evict_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1943
+ "legendFormat": "p50",
1944
+ "range": true,
1945
+ "refId": "A"
1946
+ },
1947
+ {
1948
+ "datasource": {
1949
+ "type": "prometheus",
1950
+ "uid": "prometheus"
1951
+ },
1952
+ "editorMode": "code",
1953
+ "expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=~\"vllm:kv_block_idle_before_evict_seconds_bucket|ray_vllm_kv_block_idle_before_evict_seconds_bucket\",model_name=~\"$model\",engine=~\"$engine\",runtime_name=\"__RUNTIME_NAME__\"}[$__rate_interval])))",
1954
+ "legendFormat": "p95",
1955
+ "range": true,
1956
+ "refId": "B"
1957
+ }
1958
+ ],
1959
+ "fieldConfig": {
1960
+ "defaults": {
1961
+ "unit": "s",
1962
+ "color": {
1963
+ "mode": "palette-classic"
1964
+ },
1965
+ "custom": {
1966
+ "drawStyle": "line",
1967
+ "lineInterpolation": "linear",
1968
+ "barAlignment": 0,
1969
+ "lineWidth": 1,
1970
+ "fillOpacity": 8,
1971
+ "gradientMode": "none",
1972
+ "spanNulls": false,
1973
+ "showPoints": "auto",
1974
+ "pointSize": 5,
1975
+ "stacking": {
1976
+ "mode": "none",
1977
+ "group": "A"
1978
+ },
1979
+ "axisPlacement": "auto",
1980
+ "axisLabel": "",
1981
+ "axisColorMode": "text",
1982
+ "scaleDistribution": {
1983
+ "type": "linear"
1984
+ },
1985
+ "hideFrom": {
1986
+ "tooltip": false,
1987
+ "viz": false,
1988
+ "legend": false
1989
+ },
1990
+ "thresholdsStyle": {
1991
+ "mode": "off"
1992
+ }
1993
+ },
1994
+ "thresholds": {
1995
+ "mode": "absolute",
1996
+ "steps": [
1997
+ {
1998
+ "color": "green",
1999
+ "value": null
2000
+ }
2001
+ ]
2002
+ },
2003
+ "mappings": []
2004
+ },
2005
+ "overrides": []
2006
+ },
2007
+ "options": {
2008
+ "tooltip": {
2009
+ "mode": "multi",
2010
+ "sort": "desc"
2011
+ },
2012
+ "legend": {
2013
+ "displayMode": "table",
2014
+ "placement": "bottom",
2015
+ "calcs": [
2016
+ "lastNotNull",
2017
+ "mean",
2018
+ "max"
2019
+ ]
2020
+ }
2021
+ },
2022
+ "description": "How long an evicted block sat unused first. Consistently short means blocks are being evicted while still warm.",
2023
+ "id": 21
2024
+ }
2025
+ ],
2026
+ "refresh": "30s",
2027
+ "schemaVersion": 41,
2028
+ "tags": [
2029
+ "dgx-spark",
2030
+ "generated",
2031
+ "fadenstack",
2032
+ "faden-runtime",
2033
+ "mtp",
2034
+ "prometheus",
2035
+ "vllm"
2036
+ ],
2037
+ "templating": {
2038
+ "list": [
2039
+ {
2040
+ "name": "model",
2041
+ "label": "Model",
2042
+ "type": "query",
2043
+ "datasource": {
2044
+ "type": "prometheus",
2045
+ "uid": "prometheus"
2046
+ },
2047
+ "query": "label_values({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\"}, model_name)",
2048
+ "refresh": 1,
2049
+ "sort": 1,
2050
+ "multi": true,
2051
+ "includeAll": true,
2052
+ "current": {
2053
+ "selected": true,
2054
+ "text": [
2055
+ "All"
2056
+ ],
2057
+ "value": [
2058
+ "$__all"
2059
+ ]
2060
+ },
2061
+ "allValue": ".*"
2062
+ },
2063
+ {
2064
+ "name": "engine",
2065
+ "label": "Engine",
2066
+ "type": "query",
2067
+ "datasource": {
2068
+ "type": "prometheus",
2069
+ "uid": "prometheus"
2070
+ },
2071
+ "query": "label_values({__name__=~\"vllm:generation_tokens_total|ray_vllm_generation_tokens_total\",runtime_name=\"__RUNTIME_NAME__\",model_name=~\"$model\"}, engine)",
2072
+ "refresh": 1,
2073
+ "sort": 1,
2074
+ "multi": true,
2075
+ "includeAll": true,
2076
+ "current": {
2077
+ "selected": true,
2078
+ "text": [
2079
+ "All"
2080
+ ],
2081
+ "value": [
2082
+ "$__all"
2083
+ ]
2084
+ },
2085
+ "allValue": ".*"
2086
+ }
2087
+ ]
2088
+ },
2089
+ "time": {
2090
+ "from": "now-6h",
2091
+ "to": "now"
2092
+ },
2093
+ "timepicker": {},
2094
+ "timezone": "browser",
2095
+ "title": "__TITLE__",
2096
+ "uid": "__UID__",
2097
+ "version": 1,
2098
+ "weekStart": ""
2099
+ }