vllm-benchmark-suite 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vllm_benchmark/__init__.py +9 -0
- vllm_benchmark/analysis/__init__.py +7 -0
- vllm_benchmark/analysis/diagnostics.py +585 -0
- vllm_benchmark/analysis/regression.py +307 -0
- vllm_benchmark/analysis/scoring.py +329 -0
- vllm_benchmark/cli.py +517 -0
- vllm_benchmark/config.py +132 -0
- vllm_benchmark/core/__init__.py +17 -0
- vllm_benchmark/core/benchmark.py +444 -0
- vllm_benchmark/core/metrics.py +429 -0
- vllm_benchmark/core/prompts.py +327 -0
- vllm_benchmark/core/server.py +256 -0
- vllm_benchmark/core/traffic.py +466 -0
- vllm_benchmark/reports/__init__.py +12 -0
- vllm_benchmark/reports/charts.py +364 -0
- vllm_benchmark/reports/html_report.py +289 -0
- vllm_benchmark/reports/terminal.py +197 -0
- vllm_benchmark_suite-3.0.0.dist-info/METADATA +247 -0
- vllm_benchmark_suite-3.0.0.dist-info/RECORD +22 -0
- vllm_benchmark_suite-3.0.0.dist-info/WHEEL +4 -0
- vllm_benchmark_suite-3.0.0.dist-info/entry_points.txt +2 -0
- vllm_benchmark_suite-3.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Analysis and diagnostics components."""
|
|
2
|
+
|
|
3
|
+
from vllm_benchmark.analysis.diagnostics import DiagnosticEngine
|
|
4
|
+
from vllm_benchmark.analysis.regression import RegressionDetector
|
|
5
|
+
from vllm_benchmark.analysis.scoring import VLLMScore
|
|
6
|
+
|
|
7
|
+
__all__ = ["DiagnosticEngine", "VLLMScore", "RegressionDetector"]
|
|
@@ -0,0 +1,585 @@
|
|
|
1
|
+
"""Auto-diagnostics engine for vLLM benchmark results.
|
|
2
|
+
|
|
3
|
+
Analyzes benchmark results and produces plain-English recommendations,
|
|
4
|
+
as if an expert vLLM operator were reviewing your deployment.
|
|
5
|
+
|
|
6
|
+
Author: amit
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import statistics
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class Diagnostic:
|
|
17
|
+
"""A single diagnostic finding from benchmark analysis."""
|
|
18
|
+
|
|
19
|
+
severity: str # "critical", "warning", "info", "success"
|
|
20
|
+
title: str
|
|
21
|
+
message: str
|
|
22
|
+
metric: str # which metric triggered this
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DiagnosticEngine:
|
|
26
|
+
"""Analyzes benchmark results and produces actionable diagnostics.
|
|
27
|
+
|
|
28
|
+
Each result dict is expected to contain keys such as:
|
|
29
|
+
- context_length (int)
|
|
30
|
+
- concurrent_users (int)
|
|
31
|
+
- tokens_per_second (float)
|
|
32
|
+
- avg_latency (float, seconds)
|
|
33
|
+
- max_latency (float, seconds)
|
|
34
|
+
- std_latency (float, seconds)
|
|
35
|
+
- ttft_estimate (float, seconds)
|
|
36
|
+
- throughput_per_user (float)
|
|
37
|
+
- tokens_per_watt (float, optional)
|
|
38
|
+
- failed (int)
|
|
39
|
+
- prompt_type (str, optional)
|
|
40
|
+
- gpu_stats (dict, optional) with keys:
|
|
41
|
+
- avg_gpu_util (float, 0-100)
|
|
42
|
+
- max_temperature (float, Celsius)
|
|
43
|
+
- max_mem_used (float, bytes or GB)
|
|
44
|
+
- mem_total (float, bytes or GB)
|
|
45
|
+
- cache_hit_rate (float, 0-100, optional)
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def analyze(self, results: list[dict], server_info: dict = None) -> list[Diagnostic]:
|
|
49
|
+
"""Run all diagnostic rules against results. Returns list of Diagnostic objects."""
|
|
50
|
+
if not results:
|
|
51
|
+
return [
|
|
52
|
+
Diagnostic(
|
|
53
|
+
severity="info",
|
|
54
|
+
title="No results to analyze",
|
|
55
|
+
message="No benchmark results were provided for analysis.",
|
|
56
|
+
metric="results",
|
|
57
|
+
)
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
diagnostics: list[Diagnostic] = []
|
|
61
|
+
|
|
62
|
+
diagnostics.extend(self._check_request_failures(results))
|
|
63
|
+
diagnostics.extend(self._check_high_latency_variance(results))
|
|
64
|
+
diagnostics.extend(self._check_low_gpu_utilization(results))
|
|
65
|
+
diagnostics.extend(self._check_gpu_saturated_low_throughput(results))
|
|
66
|
+
diagnostics.extend(self._check_cache_inefficiency(results))
|
|
67
|
+
diagnostics.extend(self._check_slow_ttft(results))
|
|
68
|
+
diagnostics.extend(self._check_poor_batch_scaling(results))
|
|
69
|
+
diagnostics.extend(self._check_high_temperature(results))
|
|
70
|
+
diagnostics.extend(self._check_memory_near_limit(results))
|
|
71
|
+
diagnostics.extend(self._check_excellent_throughput(results))
|
|
72
|
+
diagnostics.extend(self._check_energy_efficiency(results))
|
|
73
|
+
|
|
74
|
+
# Append vLLM config-specific recommendations when server_info is available
|
|
75
|
+
if server_info:
|
|
76
|
+
diagnostics.extend(self.advise_config(results, server_info))
|
|
77
|
+
|
|
78
|
+
has_warning_or_critical = any(
|
|
79
|
+
d.severity in ("critical", "warning") for d in diagnostics
|
|
80
|
+
)
|
|
81
|
+
if not has_warning_or_critical:
|
|
82
|
+
diagnostics.append(
|
|
83
|
+
Diagnostic(
|
|
84
|
+
severity="success",
|
|
85
|
+
title="All healthy",
|
|
86
|
+
message=(
|
|
87
|
+
"All metrics look healthy. Your vLLM deployment is "
|
|
88
|
+
"well-configured for the tested workloads."
|
|
89
|
+
),
|
|
90
|
+
metric="overall",
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
return diagnostics
|
|
95
|
+
|
|
96
|
+
def advise_config(self, results: list[dict], server_info: dict) -> list[Diagnostic]:
|
|
97
|
+
"""Produce vLLM config-specific recommendations.
|
|
98
|
+
|
|
99
|
+
Examines server configuration alongside benchmark results to
|
|
100
|
+
surface actionable tuning advice.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
results: List of result dicts from the benchmark run.
|
|
104
|
+
server_info: Dictionary of server configuration values such as
|
|
105
|
+
prefix_caching, max_num_seqs, gpu_memory_utilization,
|
|
106
|
+
tensor_parallel, quantization, etc.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
List of Diagnostic objects with config recommendations.
|
|
110
|
+
"""
|
|
111
|
+
diagnostics: list[Diagnostic] = []
|
|
112
|
+
|
|
113
|
+
prompt_types = [r.get("prompt_type", "") for r in results if r.get("prompt_type")]
|
|
114
|
+
all_random = all(pt == "random" for pt in prompt_types) if prompt_types else False
|
|
115
|
+
has_deterministic = any(
|
|
116
|
+
pt in ("deterministic", "fixed", "cached") for pt in prompt_types
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
# Rule 1: prefix caching disabled but deterministic prompts would benefit
|
|
120
|
+
prefix_caching = server_info.get("prefix_caching")
|
|
121
|
+
if not prefix_caching and has_deterministic:
|
|
122
|
+
diagnostics.append(
|
|
123
|
+
Diagnostic(
|
|
124
|
+
severity="warning",
|
|
125
|
+
title="Enable prefix caching",
|
|
126
|
+
message=(
|
|
127
|
+
"Enable --enable-prefix-caching on your vLLM server. "
|
|
128
|
+
"Your deterministic prompts would benefit significantly."
|
|
129
|
+
),
|
|
130
|
+
metric="prefix_caching",
|
|
131
|
+
)
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# Rule 2: prefix caching enabled but all prompts are random
|
|
135
|
+
if prefix_caching and all_random:
|
|
136
|
+
diagnostics.append(
|
|
137
|
+
Diagnostic(
|
|
138
|
+
severity="warning",
|
|
139
|
+
title="Prefix caching overhead with random prompts",
|
|
140
|
+
message=(
|
|
141
|
+
"You have prefix caching enabled but your prompts are "
|
|
142
|
+
"fully random — you're paying cache overhead with no "
|
|
143
|
+
"benefit. Consider disabling it for ~5-10% throughput gain."
|
|
144
|
+
),
|
|
145
|
+
metric="prefix_caching",
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# Rule 3: max_num_seqs too low for tested concurrency
|
|
150
|
+
max_num_seqs = server_info.get("max_num_seqs")
|
|
151
|
+
if max_num_seqs is not None:
|
|
152
|
+
max_users_tested = max(
|
|
153
|
+
(r.get("concurrent_users", 0) for r in results), default=0
|
|
154
|
+
)
|
|
155
|
+
if max_num_seqs < max_users_tested:
|
|
156
|
+
# Check if throughput at high concurrency is significantly lower
|
|
157
|
+
high_conc = [
|
|
158
|
+
r for r in results
|
|
159
|
+
if r.get("concurrent_users", 0) >= max_users_tested
|
|
160
|
+
and r.get("tokens_per_second") is not None
|
|
161
|
+
]
|
|
162
|
+
low_conc = [
|
|
163
|
+
r for r in results
|
|
164
|
+
if r.get("concurrent_users", 0) <= 1
|
|
165
|
+
and r.get("tokens_per_second") is not None
|
|
166
|
+
]
|
|
167
|
+
high_tps = (
|
|
168
|
+
statistics.mean([r["tokens_per_second"] for r in high_conc])
|
|
169
|
+
if high_conc else None
|
|
170
|
+
)
|
|
171
|
+
low_tps = (
|
|
172
|
+
statistics.mean([r["tokens_per_second"] for r in low_conc])
|
|
173
|
+
if low_conc else None
|
|
174
|
+
)
|
|
175
|
+
# Emit if high-concurrency throughput is not scaling well
|
|
176
|
+
if high_tps is not None and low_tps is not None and high_tps < low_tps * 1.5:
|
|
177
|
+
diagnostics.append(
|
|
178
|
+
Diagnostic(
|
|
179
|
+
severity="warning",
|
|
180
|
+
title="max_num_seqs too low for concurrency",
|
|
181
|
+
message=(
|
|
182
|
+
f"Your max_num_seqs ({max_num_seqs}) is lower than "
|
|
183
|
+
f"the concurrency you tested ({max_users_tested}). "
|
|
184
|
+
"Increase it to allow more concurrent requests."
|
|
185
|
+
),
|
|
186
|
+
metric="max_num_seqs",
|
|
187
|
+
)
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
# Rule 4: GPU memory utilization has headroom
|
|
191
|
+
gpu_mem_util = server_info.get("gpu_memory_utilization")
|
|
192
|
+
has_failures = any(r.get("failed", 0) > 0 for r in results)
|
|
193
|
+
|
|
194
|
+
if gpu_mem_util is not None and gpu_mem_util < 0.85:
|
|
195
|
+
diagnostics.append(
|
|
196
|
+
Diagnostic(
|
|
197
|
+
severity="info",
|
|
198
|
+
title="GPU memory headroom available",
|
|
199
|
+
message=(
|
|
200
|
+
f"GPU memory utilization is set to {gpu_mem_util:.0%}. "
|
|
201
|
+
"You have headroom — consider increasing to 0.90-0.95 "
|
|
202
|
+
"for better KV cache capacity."
|
|
203
|
+
),
|
|
204
|
+
metric="gpu_memory_utilization",
|
|
205
|
+
)
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
# Rule 5: GPU memory utilization very high with failures
|
|
209
|
+
if gpu_mem_util is not None and gpu_mem_util >= 0.95 and has_failures:
|
|
210
|
+
diagnostics.append(
|
|
211
|
+
Diagnostic(
|
|
212
|
+
severity="critical",
|
|
213
|
+
title="High memory utilization with failures",
|
|
214
|
+
message=(
|
|
215
|
+
f"GPU memory utilization is at {gpu_mem_util:.0%} and "
|
|
216
|
+
"you're seeing failures. Try reducing to 0.90."
|
|
217
|
+
),
|
|
218
|
+
metric="gpu_memory_utilization",
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# Rule 6: tensor parallel with low GPU utilization
|
|
223
|
+
tp = server_info.get("tensor_parallel")
|
|
224
|
+
if tp is not None and tp > 1:
|
|
225
|
+
gpu_utils = []
|
|
226
|
+
for r in results:
|
|
227
|
+
gpu = self._gpu(r)
|
|
228
|
+
util = gpu.get("avg_gpu_util")
|
|
229
|
+
if util is not None:
|
|
230
|
+
gpu_utils.append(util)
|
|
231
|
+
if gpu_utils:
|
|
232
|
+
avg_util = statistics.mean(gpu_utils)
|
|
233
|
+
if avg_util < 70:
|
|
234
|
+
diagnostics.append(
|
|
235
|
+
Diagnostic(
|
|
236
|
+
severity="warning",
|
|
237
|
+
title="Low GPU utilization with tensor parallelism",
|
|
238
|
+
message=(
|
|
239
|
+
f"With tensor_parallel={tp}, GPU utilization is "
|
|
240
|
+
f"low ({avg_util:.0f}%). If running a single GPU "
|
|
241
|
+
"is feasible, TP=1 may give better single-stream "
|
|
242
|
+
"latency."
|
|
243
|
+
),
|
|
244
|
+
metric="tensor_parallel",
|
|
245
|
+
)
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
# Rule 7: FP16/BF16 with large model (low throughput + high memory)
|
|
249
|
+
quantization = server_info.get("quantization")
|
|
250
|
+
if quantization in ("FP16", "BF16", "FP16/BF16", None):
|
|
251
|
+
tps_values = [
|
|
252
|
+
r["tokens_per_second"]
|
|
253
|
+
for r in results
|
|
254
|
+
if r.get("tokens_per_second") is not None
|
|
255
|
+
]
|
|
256
|
+
mem_usages = []
|
|
257
|
+
for r in results:
|
|
258
|
+
gpu = self._gpu(r)
|
|
259
|
+
max_mem = gpu.get("max_mem_used")
|
|
260
|
+
mem_total = gpu.get("mem_total")
|
|
261
|
+
if max_mem is not None and mem_total is not None and mem_total > 0:
|
|
262
|
+
mem_usages.append(max_mem / mem_total)
|
|
263
|
+
|
|
264
|
+
low_throughput = (
|
|
265
|
+
tps_values and statistics.mean(tps_values) < 300
|
|
266
|
+
)
|
|
267
|
+
high_memory = mem_usages and statistics.mean(mem_usages) > 0.80
|
|
268
|
+
|
|
269
|
+
if low_throughput and high_memory:
|
|
270
|
+
diagnostics.append(
|
|
271
|
+
Diagnostic(
|
|
272
|
+
severity="info",
|
|
273
|
+
title="Consider quantization",
|
|
274
|
+
message=(
|
|
275
|
+
"Consider FP8 or AWQ quantization for better "
|
|
276
|
+
"throughput with minimal quality loss."
|
|
277
|
+
),
|
|
278
|
+
metric="quantization",
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
return diagnostics
|
|
283
|
+
|
|
284
|
+
# ------------------------------------------------------------------
|
|
285
|
+
# Internal helpers
|
|
286
|
+
# ------------------------------------------------------------------
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def _ctx_label(ctx: int) -> str:
|
|
290
|
+
"""Format context length as a readable label like '32K'."""
|
|
291
|
+
if ctx >= 1_000_000:
|
|
292
|
+
return f"{ctx / 1_000_000:.0f}M"
|
|
293
|
+
return f"{ctx / 1000:.0f}K"
|
|
294
|
+
|
|
295
|
+
@staticmethod
|
|
296
|
+
def _gpu(result: dict) -> dict:
|
|
297
|
+
"""Safely extract GPU stats — supports both flat and nested formats."""
|
|
298
|
+
# v3 flattens GPU stats into the result dict
|
|
299
|
+
if "avg_gpu_util" in result:
|
|
300
|
+
return result
|
|
301
|
+
return result.get("gpu_stats") or {}
|
|
302
|
+
|
|
303
|
+
# ------------------------------------------------------------------
|
|
304
|
+
# Diagnostic rules
|
|
305
|
+
# ------------------------------------------------------------------
|
|
306
|
+
|
|
307
|
+
def _check_request_failures(self, results: list[dict]) -> list[Diagnostic]:
|
|
308
|
+
diagnostics: list[Diagnostic] = []
|
|
309
|
+
for r in results:
|
|
310
|
+
failed = r.get("failed", 0)
|
|
311
|
+
if failed and failed > 0:
|
|
312
|
+
diagnostics.append(
|
|
313
|
+
Diagnostic(
|
|
314
|
+
severity="critical",
|
|
315
|
+
title="Request failures detected",
|
|
316
|
+
message=(
|
|
317
|
+
f"{failed} requests failed during testing. Common causes: "
|
|
318
|
+
"context length exceeding max_model_len, GPU OOM, or "
|
|
319
|
+
"request timeout. Check vLLM server logs."
|
|
320
|
+
),
|
|
321
|
+
metric="failed",
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
return diagnostics
|
|
325
|
+
|
|
326
|
+
def _check_high_latency_variance(self, results: list[dict]) -> list[Diagnostic]:
|
|
327
|
+
diagnostics: list[Diagnostic] = []
|
|
328
|
+
for r in results:
|
|
329
|
+
avg_lat = r.get("avg_latency")
|
|
330
|
+
max_lat = r.get("max_latency")
|
|
331
|
+
if not avg_lat or not max_lat or avg_lat <= 0:
|
|
332
|
+
continue
|
|
333
|
+
ratio = max_lat / avg_lat
|
|
334
|
+
if ratio > 3:
|
|
335
|
+
ctx = self._ctx_label(r.get("context_length", 0))
|
|
336
|
+
users = r.get("concurrent_users", "?")
|
|
337
|
+
diagnostics.append(
|
|
338
|
+
Diagnostic(
|
|
339
|
+
severity="warning",
|
|
340
|
+
title="Extreme tail latency detected",
|
|
341
|
+
message=(
|
|
342
|
+
f"Your worst-case latency is {ratio:.1f}x your average "
|
|
343
|
+
f"at {ctx} context with {users} users. This usually means "
|
|
344
|
+
"request queuing under load. Try reducing max_num_seqs or "
|
|
345
|
+
"enabling chunked prefill."
|
|
346
|
+
),
|
|
347
|
+
metric="max_latency",
|
|
348
|
+
)
|
|
349
|
+
)
|
|
350
|
+
return diagnostics
|
|
351
|
+
|
|
352
|
+
def _check_low_gpu_utilization(self, results: list[dict]) -> list[Diagnostic]:
|
|
353
|
+
diagnostics: list[Diagnostic] = []
|
|
354
|
+
for r in results:
|
|
355
|
+
gpu = self._gpu(r)
|
|
356
|
+
util = gpu.get("avg_gpu_util")
|
|
357
|
+
if util is None:
|
|
358
|
+
continue
|
|
359
|
+
if util < 60:
|
|
360
|
+
ctx = self._ctx_label(r.get("context_length", 0))
|
|
361
|
+
users = r.get("concurrent_users", "?")
|
|
362
|
+
diagnostics.append(
|
|
363
|
+
Diagnostic(
|
|
364
|
+
severity="warning",
|
|
365
|
+
title="GPU underutilized",
|
|
366
|
+
message=(
|
|
367
|
+
f"GPU underutilized at {util:.0f}% with {users} concurrent "
|
|
368
|
+
f"users at {ctx} context. Your server may be CPU/network "
|
|
369
|
+
"bottlenecked. Consider increasing concurrent users or "
|
|
370
|
+
"batch size."
|
|
371
|
+
),
|
|
372
|
+
metric="avg_gpu_util",
|
|
373
|
+
)
|
|
374
|
+
)
|
|
375
|
+
return diagnostics
|
|
376
|
+
|
|
377
|
+
def _check_gpu_saturated_low_throughput(self, results: list[dict]) -> list[Diagnostic]:
|
|
378
|
+
diagnostics: list[Diagnostic] = []
|
|
379
|
+
tps_values = [
|
|
380
|
+
r["tokens_per_second"]
|
|
381
|
+
for r in results
|
|
382
|
+
if r.get("tokens_per_second") is not None
|
|
383
|
+
]
|
|
384
|
+
if not tps_values:
|
|
385
|
+
return diagnostics
|
|
386
|
+
median_tps = statistics.median(tps_values)
|
|
387
|
+
|
|
388
|
+
for r in results:
|
|
389
|
+
gpu = self._gpu(r)
|
|
390
|
+
util = gpu.get("avg_gpu_util")
|
|
391
|
+
tps = r.get("tokens_per_second")
|
|
392
|
+
if util is None or tps is None:
|
|
393
|
+
continue
|
|
394
|
+
if util > 90 and tps < median_tps:
|
|
395
|
+
diagnostics.append(
|
|
396
|
+
Diagnostic(
|
|
397
|
+
severity="warning",
|
|
398
|
+
title="GPU saturated with low throughput",
|
|
399
|
+
message=(
|
|
400
|
+
f"GPU is saturated ({util:.0f}%) but throughput is below "
|
|
401
|
+
"average. This may indicate memory bandwidth saturation. "
|
|
402
|
+
"Consider a smaller model or different quantization."
|
|
403
|
+
),
|
|
404
|
+
metric="tokens_per_second",
|
|
405
|
+
)
|
|
406
|
+
)
|
|
407
|
+
return diagnostics
|
|
408
|
+
|
|
409
|
+
def _check_cache_inefficiency(self, results: list[dict]) -> list[Diagnostic]:
|
|
410
|
+
diagnostics: list[Diagnostic] = []
|
|
411
|
+
for r in results:
|
|
412
|
+
gpu = self._gpu(r)
|
|
413
|
+
rate = gpu.get("cache_hit_rate")
|
|
414
|
+
if rate is None:
|
|
415
|
+
continue
|
|
416
|
+
prompt_type = r.get("prompt_type", "")
|
|
417
|
+
is_deterministic = prompt_type in ("deterministic", "fixed", "cached")
|
|
418
|
+
if rate < 20 and is_deterministic:
|
|
419
|
+
diagnostics.append(
|
|
420
|
+
Diagnostic(
|
|
421
|
+
severity="warning",
|
|
422
|
+
title="Low prefix cache hit rate",
|
|
423
|
+
message=(
|
|
424
|
+
f"Prefix cache hit rate is only {rate:.0f}% despite using "
|
|
425
|
+
"deterministic prompts. Verify --enable-prefix-caching is "
|
|
426
|
+
"set on your vLLM server."
|
|
427
|
+
),
|
|
428
|
+
metric="cache_hit_rate",
|
|
429
|
+
)
|
|
430
|
+
)
|
|
431
|
+
return diagnostics
|
|
432
|
+
|
|
433
|
+
def _check_slow_ttft(self, results: list[dict]) -> list[Diagnostic]:
|
|
434
|
+
diagnostics: list[Diagnostic] = []
|
|
435
|
+
for r in results:
|
|
436
|
+
ttft = r.get("ttft_estimate")
|
|
437
|
+
if ttft is None:
|
|
438
|
+
continue
|
|
439
|
+
ttft_ms = ttft * 1000
|
|
440
|
+
if ttft_ms > 2000:
|
|
441
|
+
diagnostics.append(
|
|
442
|
+
Diagnostic(
|
|
443
|
+
severity="warning",
|
|
444
|
+
title="Slow Time to First Token",
|
|
445
|
+
message=(
|
|
446
|
+
f"Time to First Token is {ttft_ms:.0f}ms, which will feel "
|
|
447
|
+
"sluggish to users. This may indicate KV cache pressure or "
|
|
448
|
+
"insufficient GPU memory bandwidth."
|
|
449
|
+
),
|
|
450
|
+
metric="ttft_estimate",
|
|
451
|
+
)
|
|
452
|
+
)
|
|
453
|
+
return diagnostics
|
|
454
|
+
|
|
455
|
+
def _check_poor_batch_scaling(self, results: list[dict]) -> list[Diagnostic]:
|
|
456
|
+
diagnostics: list[Diagnostic] = []
|
|
457
|
+
# Group results by context_length (and prompt_type if present)
|
|
458
|
+
groups: dict[tuple, list[dict]] = {}
|
|
459
|
+
for r in results:
|
|
460
|
+
ctx = r.get("context_length", 0)
|
|
461
|
+
pt = r.get("prompt_type", "")
|
|
462
|
+
key = (ctx, pt)
|
|
463
|
+
groups.setdefault(key, []).append(r)
|
|
464
|
+
|
|
465
|
+
for key, group in groups.items():
|
|
466
|
+
# Find single-user throughput
|
|
467
|
+
single_user = [
|
|
468
|
+
r for r in group if r.get("concurrent_users") == 1
|
|
469
|
+
]
|
|
470
|
+
if not single_user:
|
|
471
|
+
continue
|
|
472
|
+
base_tps = single_user[0].get("tokens_per_second")
|
|
473
|
+
if not base_tps or base_tps <= 0:
|
|
474
|
+
continue
|
|
475
|
+
|
|
476
|
+
for r in group:
|
|
477
|
+
users = r.get("concurrent_users", 1)
|
|
478
|
+
if users <= 1:
|
|
479
|
+
continue
|
|
480
|
+
tps = r.get("tokens_per_second")
|
|
481
|
+
if tps is None:
|
|
482
|
+
continue
|
|
483
|
+
ideal = base_tps * users
|
|
484
|
+
efficiency = (tps / ideal) * 100
|
|
485
|
+
if efficiency < 50:
|
|
486
|
+
ctx_label = self._ctx_label(key[0])
|
|
487
|
+
diagnostics.append(
|
|
488
|
+
Diagnostic(
|
|
489
|
+
severity="warning",
|
|
490
|
+
title="Poor batch scaling efficiency",
|
|
491
|
+
message=(
|
|
492
|
+
f"Batch scaling efficiency is poor at {efficiency:.0f}% "
|
|
493
|
+
f"({users} users, {ctx_label} context). Adding more "
|
|
494
|
+
"concurrent users isn't proportionally increasing "
|
|
495
|
+
"throughput. You've likely hit memory bandwidth limits."
|
|
496
|
+
),
|
|
497
|
+
metric="throughput_scaling",
|
|
498
|
+
)
|
|
499
|
+
)
|
|
500
|
+
return diagnostics
|
|
501
|
+
|
|
502
|
+
def _check_high_temperature(self, results: list[dict]) -> list[Diagnostic]:
|
|
503
|
+
diagnostics: list[Diagnostic] = []
|
|
504
|
+
for r in results:
|
|
505
|
+
gpu = self._gpu(r)
|
|
506
|
+
temp = gpu.get("max_temperature")
|
|
507
|
+
if temp is None:
|
|
508
|
+
continue
|
|
509
|
+
if temp > 85:
|
|
510
|
+
diagnostics.append(
|
|
511
|
+
Diagnostic(
|
|
512
|
+
severity="warning",
|
|
513
|
+
title="High GPU temperature",
|
|
514
|
+
message=(
|
|
515
|
+
f"GPU temperature peaked at {temp:.0f}\u00b0C. This is close "
|
|
516
|
+
"to thermal throttling thresholds and may cause inconsistent "
|
|
517
|
+
"performance."
|
|
518
|
+
),
|
|
519
|
+
metric="max_temperature",
|
|
520
|
+
)
|
|
521
|
+
)
|
|
522
|
+
return diagnostics
|
|
523
|
+
|
|
524
|
+
def _check_memory_near_limit(self, results: list[dict]) -> list[Diagnostic]:
|
|
525
|
+
diagnostics: list[Diagnostic] = []
|
|
526
|
+
for r in results:
|
|
527
|
+
gpu = self._gpu(r)
|
|
528
|
+
max_mem = gpu.get("max_mem_used")
|
|
529
|
+
mem_total = gpu.get("mem_total")
|
|
530
|
+
if max_mem is None or mem_total is None or mem_total <= 0:
|
|
531
|
+
continue
|
|
532
|
+
pct = max_mem / mem_total
|
|
533
|
+
if pct > 0.95:
|
|
534
|
+
diagnostics.append(
|
|
535
|
+
Diagnostic(
|
|
536
|
+
severity="warning",
|
|
537
|
+
title="GPU memory near limit",
|
|
538
|
+
message=(
|
|
539
|
+
f"GPU memory usage peaked at {pct * 100:.0f}%, very close "
|
|
540
|
+
"to the limit. Risk of OOM under production load. Consider "
|
|
541
|
+
"reducing max_num_seqs or using more aggressive quantization."
|
|
542
|
+
),
|
|
543
|
+
metric="max_mem_used",
|
|
544
|
+
)
|
|
545
|
+
)
|
|
546
|
+
return diagnostics
|
|
547
|
+
|
|
548
|
+
def _check_excellent_throughput(self, results: list[dict]) -> list[Diagnostic]:
|
|
549
|
+
diagnostics: list[Diagnostic] = []
|
|
550
|
+
for r in results:
|
|
551
|
+
tps = r.get("tokens_per_second")
|
|
552
|
+
if tps is not None and tps > 500:
|
|
553
|
+
ctx = self._ctx_label(r.get("context_length", 0))
|
|
554
|
+
users = r.get("concurrent_users", "?")
|
|
555
|
+
diagnostics.append(
|
|
556
|
+
Diagnostic(
|
|
557
|
+
severity="success",
|
|
558
|
+
title="Excellent throughput",
|
|
559
|
+
message=(
|
|
560
|
+
f"Excellent throughput of {tps:.0f} tokens/sec achieved "
|
|
561
|
+
f"at {ctx} context with {users} users. Your setup is "
|
|
562
|
+
"well-optimized for this workload."
|
|
563
|
+
),
|
|
564
|
+
metric="tokens_per_second",
|
|
565
|
+
)
|
|
566
|
+
)
|
|
567
|
+
return diagnostics
|
|
568
|
+
|
|
569
|
+
def _check_energy_efficiency(self, results: list[dict]) -> list[Diagnostic]:
|
|
570
|
+
diagnostics: list[Diagnostic] = []
|
|
571
|
+
for r in results:
|
|
572
|
+
tpw = r.get("tokens_per_watt")
|
|
573
|
+
if tpw is not None and tpw > 5:
|
|
574
|
+
diagnostics.append(
|
|
575
|
+
Diagnostic(
|
|
576
|
+
severity="success",
|
|
577
|
+
title="Good energy efficiency",
|
|
578
|
+
message=(
|
|
579
|
+
f"Strong energy efficiency at {tpw:.1f} tokens/watt. "
|
|
580
|
+
"Your setup is power-efficient."
|
|
581
|
+
),
|
|
582
|
+
metric="tokens_per_watt",
|
|
583
|
+
)
|
|
584
|
+
)
|
|
585
|
+
return diagnostics
|