vllm-benchmark-suite 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,9 @@
1
+ """
2
+ vLLM Benchmark Suite - The most comprehensive benchmarking tool for vLLM inference servers.
3
+
4
+ Install: uv pip install vllm-benchmark-suite
5
+ Usage: vllm-bench --quick
6
+ """
7
+
8
+ __version__ = "3.0.0"
9
+ __author__ = "amit"
@@ -0,0 +1,7 @@
1
+ """Analysis and diagnostics components."""
2
+
3
+ from vllm_benchmark.analysis.diagnostics import DiagnosticEngine
4
+ from vllm_benchmark.analysis.regression import RegressionDetector
5
+ from vllm_benchmark.analysis.scoring import VLLMScore
6
+
7
+ __all__ = ["DiagnosticEngine", "VLLMScore", "RegressionDetector"]
@@ -0,0 +1,585 @@
1
+ """Auto-diagnostics engine for vLLM benchmark results.
2
+
3
+ Analyzes benchmark results and produces plain-English recommendations,
4
+ as if an expert vLLM operator were reviewing your deployment.
5
+
6
+ Author: amit
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import statistics
12
+ from dataclasses import dataclass
13
+
14
+
15
+ @dataclass
16
+ class Diagnostic:
17
+ """A single diagnostic finding from benchmark analysis."""
18
+
19
+ severity: str # "critical", "warning", "info", "success"
20
+ title: str
21
+ message: str
22
+ metric: str # which metric triggered this
23
+
24
+
25
+ class DiagnosticEngine:
26
+ """Analyzes benchmark results and produces actionable diagnostics.
27
+
28
+ Each result dict is expected to contain keys such as:
29
+ - context_length (int)
30
+ - concurrent_users (int)
31
+ - tokens_per_second (float)
32
+ - avg_latency (float, seconds)
33
+ - max_latency (float, seconds)
34
+ - std_latency (float, seconds)
35
+ - ttft_estimate (float, seconds)
36
+ - throughput_per_user (float)
37
+ - tokens_per_watt (float, optional)
38
+ - failed (int)
39
+ - prompt_type (str, optional)
40
+ - gpu_stats (dict, optional) with keys:
41
+ - avg_gpu_util (float, 0-100)
42
+ - max_temperature (float, Celsius)
43
+ - max_mem_used (float, bytes or GB)
44
+ - mem_total (float, bytes or GB)
45
+ - cache_hit_rate (float, 0-100, optional)
46
+ """
47
+
48
+ def analyze(self, results: list[dict], server_info: dict = None) -> list[Diagnostic]:
49
+ """Run all diagnostic rules against results. Returns list of Diagnostic objects."""
50
+ if not results:
51
+ return [
52
+ Diagnostic(
53
+ severity="info",
54
+ title="No results to analyze",
55
+ message="No benchmark results were provided for analysis.",
56
+ metric="results",
57
+ )
58
+ ]
59
+
60
+ diagnostics: list[Diagnostic] = []
61
+
62
+ diagnostics.extend(self._check_request_failures(results))
63
+ diagnostics.extend(self._check_high_latency_variance(results))
64
+ diagnostics.extend(self._check_low_gpu_utilization(results))
65
+ diagnostics.extend(self._check_gpu_saturated_low_throughput(results))
66
+ diagnostics.extend(self._check_cache_inefficiency(results))
67
+ diagnostics.extend(self._check_slow_ttft(results))
68
+ diagnostics.extend(self._check_poor_batch_scaling(results))
69
+ diagnostics.extend(self._check_high_temperature(results))
70
+ diagnostics.extend(self._check_memory_near_limit(results))
71
+ diagnostics.extend(self._check_excellent_throughput(results))
72
+ diagnostics.extend(self._check_energy_efficiency(results))
73
+
74
+ # Append vLLM config-specific recommendations when server_info is available
75
+ if server_info:
76
+ diagnostics.extend(self.advise_config(results, server_info))
77
+
78
+ has_warning_or_critical = any(
79
+ d.severity in ("critical", "warning") for d in diagnostics
80
+ )
81
+ if not has_warning_or_critical:
82
+ diagnostics.append(
83
+ Diagnostic(
84
+ severity="success",
85
+ title="All healthy",
86
+ message=(
87
+ "All metrics look healthy. Your vLLM deployment is "
88
+ "well-configured for the tested workloads."
89
+ ),
90
+ metric="overall",
91
+ )
92
+ )
93
+
94
+ return diagnostics
95
+
96
+ def advise_config(self, results: list[dict], server_info: dict) -> list[Diagnostic]:
97
+ """Produce vLLM config-specific recommendations.
98
+
99
+ Examines server configuration alongside benchmark results to
100
+ surface actionable tuning advice.
101
+
102
+ Args:
103
+ results: List of result dicts from the benchmark run.
104
+ server_info: Dictionary of server configuration values such as
105
+ prefix_caching, max_num_seqs, gpu_memory_utilization,
106
+ tensor_parallel, quantization, etc.
107
+
108
+ Returns:
109
+ List of Diagnostic objects with config recommendations.
110
+ """
111
+ diagnostics: list[Diagnostic] = []
112
+
113
+ prompt_types = [r.get("prompt_type", "") for r in results if r.get("prompt_type")]
114
+ all_random = all(pt == "random" for pt in prompt_types) if prompt_types else False
115
+ has_deterministic = any(
116
+ pt in ("deterministic", "fixed", "cached") for pt in prompt_types
117
+ )
118
+
119
+ # Rule 1: prefix caching disabled but deterministic prompts would benefit
120
+ prefix_caching = server_info.get("prefix_caching")
121
+ if not prefix_caching and has_deterministic:
122
+ diagnostics.append(
123
+ Diagnostic(
124
+ severity="warning",
125
+ title="Enable prefix caching",
126
+ message=(
127
+ "Enable --enable-prefix-caching on your vLLM server. "
128
+ "Your deterministic prompts would benefit significantly."
129
+ ),
130
+ metric="prefix_caching",
131
+ )
132
+ )
133
+
134
+ # Rule 2: prefix caching enabled but all prompts are random
135
+ if prefix_caching and all_random:
136
+ diagnostics.append(
137
+ Diagnostic(
138
+ severity="warning",
139
+ title="Prefix caching overhead with random prompts",
140
+ message=(
141
+ "You have prefix caching enabled but your prompts are "
142
+ "fully random — you're paying cache overhead with no "
143
+ "benefit. Consider disabling it for ~5-10% throughput gain."
144
+ ),
145
+ metric="prefix_caching",
146
+ )
147
+ )
148
+
149
+ # Rule 3: max_num_seqs too low for tested concurrency
150
+ max_num_seqs = server_info.get("max_num_seqs")
151
+ if max_num_seqs is not None:
152
+ max_users_tested = max(
153
+ (r.get("concurrent_users", 0) for r in results), default=0
154
+ )
155
+ if max_num_seqs < max_users_tested:
156
+ # Check if throughput at high concurrency is significantly lower
157
+ high_conc = [
158
+ r for r in results
159
+ if r.get("concurrent_users", 0) >= max_users_tested
160
+ and r.get("tokens_per_second") is not None
161
+ ]
162
+ low_conc = [
163
+ r for r in results
164
+ if r.get("concurrent_users", 0) <= 1
165
+ and r.get("tokens_per_second") is not None
166
+ ]
167
+ high_tps = (
168
+ statistics.mean([r["tokens_per_second"] for r in high_conc])
169
+ if high_conc else None
170
+ )
171
+ low_tps = (
172
+ statistics.mean([r["tokens_per_second"] for r in low_conc])
173
+ if low_conc else None
174
+ )
175
+ # Emit if high-concurrency throughput is not scaling well
176
+ if high_tps is not None and low_tps is not None and high_tps < low_tps * 1.5:
177
+ diagnostics.append(
178
+ Diagnostic(
179
+ severity="warning",
180
+ title="max_num_seqs too low for concurrency",
181
+ message=(
182
+ f"Your max_num_seqs ({max_num_seqs}) is lower than "
183
+ f"the concurrency you tested ({max_users_tested}). "
184
+ "Increase it to allow more concurrent requests."
185
+ ),
186
+ metric="max_num_seqs",
187
+ )
188
+ )
189
+
190
+ # Rule 4: GPU memory utilization has headroom
191
+ gpu_mem_util = server_info.get("gpu_memory_utilization")
192
+ has_failures = any(r.get("failed", 0) > 0 for r in results)
193
+
194
+ if gpu_mem_util is not None and gpu_mem_util < 0.85:
195
+ diagnostics.append(
196
+ Diagnostic(
197
+ severity="info",
198
+ title="GPU memory headroom available",
199
+ message=(
200
+ f"GPU memory utilization is set to {gpu_mem_util:.0%}. "
201
+ "You have headroom — consider increasing to 0.90-0.95 "
202
+ "for better KV cache capacity."
203
+ ),
204
+ metric="gpu_memory_utilization",
205
+ )
206
+ )
207
+
208
+ # Rule 5: GPU memory utilization very high with failures
209
+ if gpu_mem_util is not None and gpu_mem_util >= 0.95 and has_failures:
210
+ diagnostics.append(
211
+ Diagnostic(
212
+ severity="critical",
213
+ title="High memory utilization with failures",
214
+ message=(
215
+ f"GPU memory utilization is at {gpu_mem_util:.0%} and "
216
+ "you're seeing failures. Try reducing to 0.90."
217
+ ),
218
+ metric="gpu_memory_utilization",
219
+ )
220
+ )
221
+
222
+ # Rule 6: tensor parallel with low GPU utilization
223
+ tp = server_info.get("tensor_parallel")
224
+ if tp is not None and tp > 1:
225
+ gpu_utils = []
226
+ for r in results:
227
+ gpu = self._gpu(r)
228
+ util = gpu.get("avg_gpu_util")
229
+ if util is not None:
230
+ gpu_utils.append(util)
231
+ if gpu_utils:
232
+ avg_util = statistics.mean(gpu_utils)
233
+ if avg_util < 70:
234
+ diagnostics.append(
235
+ Diagnostic(
236
+ severity="warning",
237
+ title="Low GPU utilization with tensor parallelism",
238
+ message=(
239
+ f"With tensor_parallel={tp}, GPU utilization is "
240
+ f"low ({avg_util:.0f}%). If running a single GPU "
241
+ "is feasible, TP=1 may give better single-stream "
242
+ "latency."
243
+ ),
244
+ metric="tensor_parallel",
245
+ )
246
+ )
247
+
248
+ # Rule 7: FP16/BF16 with large model (low throughput + high memory)
249
+ quantization = server_info.get("quantization")
250
+ if quantization in ("FP16", "BF16", "FP16/BF16", None):
251
+ tps_values = [
252
+ r["tokens_per_second"]
253
+ for r in results
254
+ if r.get("tokens_per_second") is not None
255
+ ]
256
+ mem_usages = []
257
+ for r in results:
258
+ gpu = self._gpu(r)
259
+ max_mem = gpu.get("max_mem_used")
260
+ mem_total = gpu.get("mem_total")
261
+ if max_mem is not None and mem_total is not None and mem_total > 0:
262
+ mem_usages.append(max_mem / mem_total)
263
+
264
+ low_throughput = (
265
+ tps_values and statistics.mean(tps_values) < 300
266
+ )
267
+ high_memory = mem_usages and statistics.mean(mem_usages) > 0.80
268
+
269
+ if low_throughput and high_memory:
270
+ diagnostics.append(
271
+ Diagnostic(
272
+ severity="info",
273
+ title="Consider quantization",
274
+ message=(
275
+ "Consider FP8 or AWQ quantization for better "
276
+ "throughput with minimal quality loss."
277
+ ),
278
+ metric="quantization",
279
+ )
280
+ )
281
+
282
+ return diagnostics
283
+
284
+ # ------------------------------------------------------------------
285
+ # Internal helpers
286
+ # ------------------------------------------------------------------
287
+
288
+ @staticmethod
289
+ def _ctx_label(ctx: int) -> str:
290
+ """Format context length as a readable label like '32K'."""
291
+ if ctx >= 1_000_000:
292
+ return f"{ctx / 1_000_000:.0f}M"
293
+ return f"{ctx / 1000:.0f}K"
294
+
295
+ @staticmethod
296
+ def _gpu(result: dict) -> dict:
297
+ """Safely extract GPU stats — supports both flat and nested formats."""
298
+ # v3 flattens GPU stats into the result dict
299
+ if "avg_gpu_util" in result:
300
+ return result
301
+ return result.get("gpu_stats") or {}
302
+
303
+ # ------------------------------------------------------------------
304
+ # Diagnostic rules
305
+ # ------------------------------------------------------------------
306
+
307
+ def _check_request_failures(self, results: list[dict]) -> list[Diagnostic]:
308
+ diagnostics: list[Diagnostic] = []
309
+ for r in results:
310
+ failed = r.get("failed", 0)
311
+ if failed and failed > 0:
312
+ diagnostics.append(
313
+ Diagnostic(
314
+ severity="critical",
315
+ title="Request failures detected",
316
+ message=(
317
+ f"{failed} requests failed during testing. Common causes: "
318
+ "context length exceeding max_model_len, GPU OOM, or "
319
+ "request timeout. Check vLLM server logs."
320
+ ),
321
+ metric="failed",
322
+ )
323
+ )
324
+ return diagnostics
325
+
326
+ def _check_high_latency_variance(self, results: list[dict]) -> list[Diagnostic]:
327
+ diagnostics: list[Diagnostic] = []
328
+ for r in results:
329
+ avg_lat = r.get("avg_latency")
330
+ max_lat = r.get("max_latency")
331
+ if not avg_lat or not max_lat or avg_lat <= 0:
332
+ continue
333
+ ratio = max_lat / avg_lat
334
+ if ratio > 3:
335
+ ctx = self._ctx_label(r.get("context_length", 0))
336
+ users = r.get("concurrent_users", "?")
337
+ diagnostics.append(
338
+ Diagnostic(
339
+ severity="warning",
340
+ title="Extreme tail latency detected",
341
+ message=(
342
+ f"Your worst-case latency is {ratio:.1f}x your average "
343
+ f"at {ctx} context with {users} users. This usually means "
344
+ "request queuing under load. Try reducing max_num_seqs or "
345
+ "enabling chunked prefill."
346
+ ),
347
+ metric="max_latency",
348
+ )
349
+ )
350
+ return diagnostics
351
+
352
+ def _check_low_gpu_utilization(self, results: list[dict]) -> list[Diagnostic]:
353
+ diagnostics: list[Diagnostic] = []
354
+ for r in results:
355
+ gpu = self._gpu(r)
356
+ util = gpu.get("avg_gpu_util")
357
+ if util is None:
358
+ continue
359
+ if util < 60:
360
+ ctx = self._ctx_label(r.get("context_length", 0))
361
+ users = r.get("concurrent_users", "?")
362
+ diagnostics.append(
363
+ Diagnostic(
364
+ severity="warning",
365
+ title="GPU underutilized",
366
+ message=(
367
+ f"GPU underutilized at {util:.0f}% with {users} concurrent "
368
+ f"users at {ctx} context. Your server may be CPU/network "
369
+ "bottlenecked. Consider increasing concurrent users or "
370
+ "batch size."
371
+ ),
372
+ metric="avg_gpu_util",
373
+ )
374
+ )
375
+ return diagnostics
376
+
377
+ def _check_gpu_saturated_low_throughput(self, results: list[dict]) -> list[Diagnostic]:
378
+ diagnostics: list[Diagnostic] = []
379
+ tps_values = [
380
+ r["tokens_per_second"]
381
+ for r in results
382
+ if r.get("tokens_per_second") is not None
383
+ ]
384
+ if not tps_values:
385
+ return diagnostics
386
+ median_tps = statistics.median(tps_values)
387
+
388
+ for r in results:
389
+ gpu = self._gpu(r)
390
+ util = gpu.get("avg_gpu_util")
391
+ tps = r.get("tokens_per_second")
392
+ if util is None or tps is None:
393
+ continue
394
+ if util > 90 and tps < median_tps:
395
+ diagnostics.append(
396
+ Diagnostic(
397
+ severity="warning",
398
+ title="GPU saturated with low throughput",
399
+ message=(
400
+ f"GPU is saturated ({util:.0f}%) but throughput is below "
401
+ "average. This may indicate memory bandwidth saturation. "
402
+ "Consider a smaller model or different quantization."
403
+ ),
404
+ metric="tokens_per_second",
405
+ )
406
+ )
407
+ return diagnostics
408
+
409
+ def _check_cache_inefficiency(self, results: list[dict]) -> list[Diagnostic]:
410
+ diagnostics: list[Diagnostic] = []
411
+ for r in results:
412
+ gpu = self._gpu(r)
413
+ rate = gpu.get("cache_hit_rate")
414
+ if rate is None:
415
+ continue
416
+ prompt_type = r.get("prompt_type", "")
417
+ is_deterministic = prompt_type in ("deterministic", "fixed", "cached")
418
+ if rate < 20 and is_deterministic:
419
+ diagnostics.append(
420
+ Diagnostic(
421
+ severity="warning",
422
+ title="Low prefix cache hit rate",
423
+ message=(
424
+ f"Prefix cache hit rate is only {rate:.0f}% despite using "
425
+ "deterministic prompts. Verify --enable-prefix-caching is "
426
+ "set on your vLLM server."
427
+ ),
428
+ metric="cache_hit_rate",
429
+ )
430
+ )
431
+ return diagnostics
432
+
433
+ def _check_slow_ttft(self, results: list[dict]) -> list[Diagnostic]:
434
+ diagnostics: list[Diagnostic] = []
435
+ for r in results:
436
+ ttft = r.get("ttft_estimate")
437
+ if ttft is None:
438
+ continue
439
+ ttft_ms = ttft * 1000
440
+ if ttft_ms > 2000:
441
+ diagnostics.append(
442
+ Diagnostic(
443
+ severity="warning",
444
+ title="Slow Time to First Token",
445
+ message=(
446
+ f"Time to First Token is {ttft_ms:.0f}ms, which will feel "
447
+ "sluggish to users. This may indicate KV cache pressure or "
448
+ "insufficient GPU memory bandwidth."
449
+ ),
450
+ metric="ttft_estimate",
451
+ )
452
+ )
453
+ return diagnostics
454
+
455
+ def _check_poor_batch_scaling(self, results: list[dict]) -> list[Diagnostic]:
456
+ diagnostics: list[Diagnostic] = []
457
+ # Group results by context_length (and prompt_type if present)
458
+ groups: dict[tuple, list[dict]] = {}
459
+ for r in results:
460
+ ctx = r.get("context_length", 0)
461
+ pt = r.get("prompt_type", "")
462
+ key = (ctx, pt)
463
+ groups.setdefault(key, []).append(r)
464
+
465
+ for key, group in groups.items():
466
+ # Find single-user throughput
467
+ single_user = [
468
+ r for r in group if r.get("concurrent_users") == 1
469
+ ]
470
+ if not single_user:
471
+ continue
472
+ base_tps = single_user[0].get("tokens_per_second")
473
+ if not base_tps or base_tps <= 0:
474
+ continue
475
+
476
+ for r in group:
477
+ users = r.get("concurrent_users", 1)
478
+ if users <= 1:
479
+ continue
480
+ tps = r.get("tokens_per_second")
481
+ if tps is None:
482
+ continue
483
+ ideal = base_tps * users
484
+ efficiency = (tps / ideal) * 100
485
+ if efficiency < 50:
486
+ ctx_label = self._ctx_label(key[0])
487
+ diagnostics.append(
488
+ Diagnostic(
489
+ severity="warning",
490
+ title="Poor batch scaling efficiency",
491
+ message=(
492
+ f"Batch scaling efficiency is poor at {efficiency:.0f}% "
493
+ f"({users} users, {ctx_label} context). Adding more "
494
+ "concurrent users isn't proportionally increasing "
495
+ "throughput. You've likely hit memory bandwidth limits."
496
+ ),
497
+ metric="throughput_scaling",
498
+ )
499
+ )
500
+ return diagnostics
501
+
502
+ def _check_high_temperature(self, results: list[dict]) -> list[Diagnostic]:
503
+ diagnostics: list[Diagnostic] = []
504
+ for r in results:
505
+ gpu = self._gpu(r)
506
+ temp = gpu.get("max_temperature")
507
+ if temp is None:
508
+ continue
509
+ if temp > 85:
510
+ diagnostics.append(
511
+ Diagnostic(
512
+ severity="warning",
513
+ title="High GPU temperature",
514
+ message=(
515
+ f"GPU temperature peaked at {temp:.0f}\u00b0C. This is close "
516
+ "to thermal throttling thresholds and may cause inconsistent "
517
+ "performance."
518
+ ),
519
+ metric="max_temperature",
520
+ )
521
+ )
522
+ return diagnostics
523
+
524
+ def _check_memory_near_limit(self, results: list[dict]) -> list[Diagnostic]:
525
+ diagnostics: list[Diagnostic] = []
526
+ for r in results:
527
+ gpu = self._gpu(r)
528
+ max_mem = gpu.get("max_mem_used")
529
+ mem_total = gpu.get("mem_total")
530
+ if max_mem is None or mem_total is None or mem_total <= 0:
531
+ continue
532
+ pct = max_mem / mem_total
533
+ if pct > 0.95:
534
+ diagnostics.append(
535
+ Diagnostic(
536
+ severity="warning",
537
+ title="GPU memory near limit",
538
+ message=(
539
+ f"GPU memory usage peaked at {pct * 100:.0f}%, very close "
540
+ "to the limit. Risk of OOM under production load. Consider "
541
+ "reducing max_num_seqs or using more aggressive quantization."
542
+ ),
543
+ metric="max_mem_used",
544
+ )
545
+ )
546
+ return diagnostics
547
+
548
+ def _check_excellent_throughput(self, results: list[dict]) -> list[Diagnostic]:
549
+ diagnostics: list[Diagnostic] = []
550
+ for r in results:
551
+ tps = r.get("tokens_per_second")
552
+ if tps is not None and tps > 500:
553
+ ctx = self._ctx_label(r.get("context_length", 0))
554
+ users = r.get("concurrent_users", "?")
555
+ diagnostics.append(
556
+ Diagnostic(
557
+ severity="success",
558
+ title="Excellent throughput",
559
+ message=(
560
+ f"Excellent throughput of {tps:.0f} tokens/sec achieved "
561
+ f"at {ctx} context with {users} users. Your setup is "
562
+ "well-optimized for this workload."
563
+ ),
564
+ metric="tokens_per_second",
565
+ )
566
+ )
567
+ return diagnostics
568
+
569
+ def _check_energy_efficiency(self, results: list[dict]) -> list[Diagnostic]:
570
+ diagnostics: list[Diagnostic] = []
571
+ for r in results:
572
+ tpw = r.get("tokens_per_watt")
573
+ if tpw is not None and tpw > 5:
574
+ diagnostics.append(
575
+ Diagnostic(
576
+ severity="success",
577
+ title="Good energy efficiency",
578
+ message=(
579
+ f"Strong energy efficiency at {tpw:.1f} tokens/watt. "
580
+ "Your setup is power-efficient."
581
+ ),
582
+ metric="tokens_per_watt",
583
+ )
584
+ )
585
+ return diagnostics